- docstring: null function: setup.read - docstring: null function: test_structured_cov_estimator.TestStructuredCovEstimator - docstring: null function: test_structured_cov_estimator.test_random_covariance - docstring: null function: test_structured_cov_estimator.test_nan_option_covariance - docstring: null function: test_structured_cov_estimator.test_decompose_covariance - docstring: null function: test_structured_cov_estimator.test_constructed_covariance - docstring: ' train model Returns-------pred_score: pandas.DataFramepredict scoresperformance: dictmodel performance' function: test_all_pipeline.train - docstring: ' A fake experiment workflow to test uri Returns-------pass_or_not_for_default_uri: boolpass_or_not_for_current_uri: booltemporary_exp_dir: str' function: test_all_pipeline.fake_experiment - docstring: ' backtest and analysis Parameters----------rid : strthe id of the recorder to be used in this functionuri_path: strmlflow uri pathReturns-------analysis : pandas.DataFramethe analysis result' function: test_all_pipeline.backtest_analysis - docstring: null function: test_all_pipeline.TestAllFlow - docstring: null function: test_all_pipeline.tearDownClass - docstring: null function: test_all_pipeline.test_0_train - docstring: null function: test_all_pipeline.test_1_backtest - docstring: null function: test_all_pipeline.test_2_expmanager - docstring: null function: test_pit.TestPIT - docstring: null function: test_pit.tearDownClass - docstring: null function: test_pit.setUpClass - docstring: null function: test_pit.setUp - docstring: null function: test_pit.to_str - docstring: null function: test_pit.check_same - docstring: " \nself.check_same(data.describe(), res)res = " function: test_pit.test_query - docstring: " \nself.check_same(data, expect)@pytest.mark.slow" function: test_pit.test_no_exist_data - docstring: " \nself.check_same(data.tail(15), expect)" function: test_pit.test_expr - docstring: " \nself.check_same(s[~s.duplicated().values], expect)" function: test_pit.test_unlimit - docstring: " \nself.check_same(data, except_data)" function: test_pit.test_expr2 - docstring: null function: test_get_data.TestGetData - docstring: null function: test_get_data.setUpClass - docstring: null function: test_get_data.tearDownClass - docstring: null function: test_get_data.test_0_qlib_data - docstring: null function: test_contrib_model.TestAllFlow - docstring: null function: test_contrib_model.test_0_initialize - docstring: null function: test_contrib_workflow.train_multiseg - docstring: null function: test_contrib_workflow.train_mse - docstring: null function: test_contrib_workflow.TestAllFlow - docstring: null function: test_contrib_workflow.tearDownClass - docstring: null function: test_contrib_workflow.test_0_multiseg - docstring: null function: test_contrib_workflow.test_1_mse - docstring: null function: test_workflow.WorkflowTest - docstring: null function: test_workflow.tearDown - docstring: null function: test_dump_data.TestDumpData - docstring: null function: test_dump_data.setUpClass - docstring: null function: test_dump_data.tearDownClass - docstring: null function: test_dump_data.test_0_dump_bin - docstring: null function: test_dump_data.test_1_dump_calendars - docstring: null function: test_dump_data.test_2_dump_instruments - docstring: null function: test_dump_data.test_3_dump_features - docstring: ' Feature First Difference Parameters----------feature : Expressionfeature instanceReturns----------Expressiona feature instance with first difference' function: test_register_ops.Diff - docstring: null function: test_register_ops._load_internal - docstring: null function: test_register_ops.get_extended_window_size - docstring: ' Feature Distance Parameters----------feature : Expressionfeature instanceReturns----------Expressiona feature instance with distance' function: test_register_ops.Distance - docstring: null function: test_register_ops._load_internal - docstring: null function: test_register_ops.TestRegiterCustomOps - docstring: null function: test_register_ops.setUpClass - docstring: null function: test_datalayer.TestDataset - docstring: null function: test_datalayer.testCSI300 - docstring: null function: test_processor.TestProcessor - docstring: null function: test_processor.test_MinMaxNorm - docstring: null function: test_processor.normalize - docstring: null function: test_processor.test_ZScoreNorm - docstring: null function: test_processor.normalize - docstring: null function: test_processor.test_CSZFillna - docstring: null function: test_handler.HandlerTests - docstring: null function: test_handler.to_str - docstring: null function: test_handler_storage.TestHandler - docstring: null function: test_handler_storage.get_feature_config - docstring: null function: test_handler_storage.TestHandlerStorage - docstring: null function: test_dataset.TestDataset - docstring: null function: test_saoe_simple.test_pickle_data_inspect - docstring: null function: test_saoe_simple.test_simulator_first_step - docstring: null function: test_saoe_simple.test_simulator_stop_twap - docstring: null function: test_saoe_simple.test_simulator_stop_early - docstring: null function: test_saoe_simple.test_simulator_start_middle - docstring: null function: test_saoe_simple.test_interpreter - docstring: null function: test_saoe_simple.EmulateEnvWrapper - docstring: null function: test_saoe_simple.test_network_sanity - docstring: null function: test_saoe_simple.EmulateEnvWrapper - docstring: null function: test_saoe_simple.test_twap_strategy - docstring: null function: test_saoe_simple.test_cn_ppo_strategy - docstring: null function: test_data_queue.DummyDataset - docstring: null function: test_data_queue._worker - docstring: null function: test_data_queue._queue_to_list - docstring: null function: test_data_queue.test_pytorch_dataloader - docstring: null function: test_data_queue.test_multiprocess_shared_dataloader - docstring: null function: test_data_queue.test_exit_on_crash_finite - docstring: null function: test_data_queue._exit_finite - docstring: null function: test_data_queue.test_exit_on_crash_infinite - docstring: null function: test_qlib_simulator.is_close - docstring: null function: test_qlib_simulator.get_order - docstring: null function: test_qlib_simulator.get_configs - docstring: null function: test_qlib_simulator.get_simulator - docstring: null function: test_qlib_simulator.test_simulator_first_step - docstring: null function: test_qlib_simulator.test_simulator_stop_twap - docstring: null function: test_logger.SimpleEnv - docstring: null function: test_logger.reset - docstring: null function: test_logger.step - docstring: null function: test_logger.render - docstring: null function: test_logger.AnyPolicy - docstring: null function: test_logger.forward - docstring: null function: test_logger.learn - docstring: null function: test_logger.test_simple_env_logger - docstring: null function: test_logger.SimpleSimulator - docstring: null function: test_logger.step - docstring: null function: test_logger.get_state - docstring: null function: test_logger.done - docstring: null function: test_logger.DummyStateInterpreter - docstring: null function: test_logger.interpret - docstring: null function: test_logger.observation_space - docstring: null function: test_logger.DummyActionInterpreter - docstring: null function: test_logger.interpret - docstring: null function: test_logger.action_space - docstring: null function: test_logger.RandomFivePolicy - docstring: null function: test_logger.forward - docstring: null function: test_logger.learn - docstring: null function: test_trainer.ZeroSimulator - docstring: null function: test_trainer.step - docstring: null function: test_trainer.get_state - docstring: null function: test_trainer.done - docstring: null function: test_trainer.NoopStateInterpreter - docstring: null function: test_trainer.interpret - docstring: null function: test_trainer.NoopActionInterpreter - docstring: null function: test_trainer.interpret - docstring: null function: test_trainer.AccReward - docstring: null function: test_trainer.reward - docstring: null function: test_trainer.PolicyNet - docstring: null function: test_trainer.forward - docstring: null function: test_trainer._ppo_policy - docstring: null function: test_trainer.test_trainer - docstring: null function: test_trainer.test_trainer_fast_dev_run - docstring: null function: test_trainer.test_trainer_earlystop - docstring: null function: test_finite_env.FiniteEnv - docstring: null function: test_finite_env.reset - docstring: null function: test_finite_env.step - docstring: null function: test_finite_env.FiniteEnvWithComplexObs - docstring: null function: test_finite_env.reset - docstring: null function: test_finite_env.step - docstring: null function: test_finite_env.DummyDataset - docstring: null function: test_finite_env.AnyPolicy - docstring: null function: test_finite_env.forward - docstring: null function: test_finite_env.learn - docstring: null function: test_finite_env._finite_env_factory - docstring: null function: test_finite_env.MetricTracker - docstring: null function: test_finite_env.on_env_step - docstring: null function: test_finite_env.validate - docstring: null function: test_finite_env.DoNothingTracker - docstring: null function: test_finite_env.on_env_step - docstring: null function: test_finite_env.test_finite_dummy_vector_env - docstring: null function: test_finite_env.test_finite_shmem_vector_env - docstring: null function: test_finite_env.test_finite_subproc_vector_env - docstring: null function: test_finite_env.test_nan - docstring: null function: test_finite_env.test_finite_dummy_vector_env_complex - docstring: null function: test_update_pred.TestRolling - docstring: " \nThis test is for testing if it will raise error if the `to_date`\ \ is out of the boundary." function: test_update_pred.test_update_pred - docstring: null function: test_file_strategy.FileStrTest - docstring: null function: test_file_strategy._gen_orders - docstring: null function: test_high_freq_trading.TestHFBacktest - docstring: null function: test_high_freq_trading.setUpClass - docstring: null function: test_high_freq_trading._gen_orders - docstring: null function: test_utils.SingletonTest - docstring: null function: test_utils.test_singleton - docstring: null function: test_sumarize.TestSummarize - docstring: null function: test_sumarize.test_chat - docstring: null function: test_sumarize.test_execution - docstring: null function: test_sumarize.test_generate_batch_result - docstring: null function: test_sumarize.test_parse2txt - docstring: null function: test_cfg.FincoTpl - docstring: ' Motivation: make sure the configuable template is consistent with the default config tpl_p = get_tpl_path()with (tpl_p / "sl" / "workflow_config.yaml").open("rb") as fp:config = yaml.safe_load(fp)# init_data_handlerhd: DataHandlerLP = init_instance_by_config(config["task"]["dataset"]["kwargs"]["handler"])# NOTE: The config in workflow_config.yaml is generated by the following code:# dump in yaml format to file without auto linebreak# print(yaml.dump(hd.data_loader.fields, width=10000, stream=open("_tmp", "w")))with (tpl_p / "sl-cfg" / "workflow_config.yaml").open("rb") as fp:config = yaml.safe_load(fp)hd_ds: DataHandlerLP = init_instance_by_config(config["task"]["dataset"]["kwargs"]["handler"])self.assertEqual(hd_ds.data_loader.fields, hd.data_loader.fields)check = hd_ds.fetch().fillna(0.0) == hd.fetch().fillna(0.0)self.assertTrue(check.all().all())' function: test_cfg.test_tpl_consistence - docstring: null function: test_mlflow.MLflowTest - docstring: null function: test_mlflow.tearDown - docstring: " \nPlease refer to qlib/workflow/expm.py:MLflowExpManager._clientwe\ \ don't cache _client (this is helpful to reduce maintainance work when MLflowExpManager's\ \ uri is chagned)This implementation is based on the assumption creating a client\ \ is fast" function: test_mlflow.test_creating_client - docstring: null function: test_elem_operator.TestElementOperator - docstring: null function: test_elem_operator.setUp - docstring: null function: test_elem_operator.test_Abs - docstring: null function: test_elem_operator.test_Sign - docstring: null function: test_elem_operator.TestOperatorDataSetting - docstring: null function: test_elem_operator.test_setting - docstring: null function: test_elem_operator.TestInstElementOperator - docstring: null function: test_elem_operator.setUp - docstring: null function: test_special_ops.TestOperatorDataSetting - docstring: null function: test_special_ops.test_setting - docstring: null function: test_special_ops.test_case2 - docstring: " \nSample raw calendar into calendar with sam_minutes freq, shift\ \ represents the shift minute the market time- open time of stock market is [9:30\ \ - shift*pd.Timedelta(minutes=1)]- mid close time of stock market is [11:29 -\ \ shift*pd.Timedelta(minutes=1)]- mid open time of stock market is [13:00 - shift*pd.Timedelta(minutes=1)]-\ \ close time of stock market is [14:59 - shift*pd.Timedelta(minutes=1)]" function: test_utils.cal_sam_minute - docstring: null function: test_utils.TimeUtils - docstring: null function: test_utils.setUpClass - docstring: null function: test_utils.test_cal_sam_minute - docstring: null function: test_index_data.IndexDataTest - docstring: null function: test_index_data.test_index_single_data - docstring: null function: test_index_data.test_index_multi_data - docstring: null function: test_index_data.test_sorting - docstring: null function: test_index_data.test_corner_cases - docstring: null function: test_index_data.test_ops - docstring: null function: test_index_data.test_todo - docstring: null function: test_get_multi_proc.get_features - docstring: null function: test_get_multi_proc.TestGetData - docstring: " \nFor testing if it will raise error" function: test_get_multi_proc.test_multi_proc - docstring: null function: test_sepdf.SepDF - docstring: null function: test_sepdf.to_str - docstring: null function: test_storage.TestStorage - docstring: null function: test_storage.test_calendar_storage - docstring: " \nThe meaning of instrument, such as CSI500:CSI500 composition\ \ changes:date add remove2005-01-01 SH6000002005-01-01\ \ SH6000012005-01-01 SH6000022005-02-01 SH600003 SH6000002005-02-15\ \ SH600000 SH600002Calendar:pd.date_range(start=\"2020-01-01\", stop=\"\ 2020-03-01\", freq=\"1D\")Instrument:symbol start_time end_timeSH600000\ \ 2005-01-01 2005-01-31 (2005-02-01 Last trading day)SH600000 2005-02-15\ \ 2005-03-01SH600001 2005-01-01 2005-03-01SH600002 2005-01-01\ \ 2005-02-14 (2005-02-15 Last trading day)SH600003 2005-02-01 2005-03-01InstrumentStorage:{\"\ SH600000\": [(2005-01-01, 2005-01-31), (2005-02-15, 2005-03-01)],\"SH600001\"\ : [(2005-01-01, 2005-03-01)],\"SH600002\": [(2005-01-01, 2005-02-14)],\"SH600003\"\ : [(2005-02-01, 2005-03-01)],}" function: test_storage.test_instrument_storage - docstring: " \nCalendar:pd.date_range(start=\"2005-01-01\", stop=\"2005-03-01\"\ , freq=\"1D\")Instrument:{\"SH600000\": [(2005-01-01, 2005-01-31), (2005-02-15,\ \ 2005-03-01)],\"SH600001\": [(2005-01-01, 2005-03-01)],\"SH600002\": [(2005-01-01,\ \ 2005-02-14)],\"SH600003\": [(2005-02-01, 2005-03-01)],}Feature:Stock data(close):2005-01-01\ \ ... 2005-02-01 ... 2005-02-14 2005-02-15 ... 2005-03-01SH600000 \ \ 1 ... 3 ... 4 5 6SH600001\ \ 1 ... 4 ... 5 6 7SH600002\ \ 1 ... 5 ... 6 nan nanSH600003\ \ nan ... 1 ... 2 3 4FeatureStorage(SH600000,\ \ close):[(calendar.index(\"2005-01-01\"), 1),...,(calendar.index(\"2005-03-01\"\ ), 6)]====> [(0, 1), ..., (59, 6)]FeatureStorage(SH600002, close):[(calendar.index(\"\ 2005-01-01\"), 1),...,(calendar.index(\"2005-02-14\"), 6)]===> [(0, 1), ..., (44,\ \ 6)]FeatureStorage(SH600003, close):[(calendar.index(\"2005-02-01\"), 1),...,(calendar.index(\"\ 2005-03-01\"), 4)]===> [(31, 1), ..., (59, 4)]" function: test_storage.test_feature_storage - docstring: null function: run_all_model.only_allow_defined_args - docstring: ' Internal wrapper function. argspec = inspect.getfullargspec(function_to_decorate)valid_names = set(argspec.args + argspec.kwonlyargs)if "self" in valid_names:valid_names.remove("self")for arg_name in kwargs:if arg_name not in valid_names:raise ValueError("Unknown argument seen ''%s'', expected: [%s]" % (arg_name, ", ".join(valid_names)))return function_to_decorate(*args, **kwargs)return _return_wrapped# function to handle ctrl z and ctrl c' function: run_all_model._return_wrapped - docstring: null function: run_all_model.handler - docstring: null function: run_all_model.cal_mean_std - docstring: null function: run_all_model.create_env - docstring: null function: run_all_model.execute - docstring: null function: run_all_model.get_all_folders - docstring: null function: run_all_model.get_all_files - docstring: null function: run_all_model.get_all_results - docstring: null function: run_all_model.gen_and_save_md_table - docstring: null function: run_all_model.gen_yaml_file_without_seed_kwargs - docstring: null function: 'run_all_model.ModelRunner:' - docstring: null function: run_all_model._init_qlib - docstring: " \nPlease be aware that this function can only work under Linux.\ \ MacOS and Windows will be supported in the future.Any PR to enhance this method\ \ is highly welcomed. Besides, this script doesn't support parallel running the\ \ same modelfor multiple times, and this will be fixed in the future development.Parameters:-----------times\ \ : intdetermines how many times the model should be running.models : str or listdetermines\ \ the specific model or list of models to run or exclude.exclude : booleandetermines\ \ whether the model being used is excluded or included.dataset : strdetermines\ \ the dataset to be used for each model.universe : strthe stock universe of the\ \ dataset.default \"\" indicates thatqlib_uri : strthe uri to install qlib with\ \ pipit could be URI on the remote or local path (NOTE: the local path must be\ \ an absolute path)exp_folder_name: strthe name of the experiment folderwait_before_rm_env\ \ : boolwait before remove environment.wait_when_err : boolwait when errors raised\ \ when executing commandsUsage:-------Here are some use cases of the function\ \ in the bash:The run_all_models will decide which config to run based no `models`\ \ `dataset` `universe`Example 1):models=\"lightgbm\", dataset=\"Alpha158\", universe=\"\ \" will result in running the following configexamples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yamlmodels=\"\ lightgbm\", dataset=\"Alpha158\", universe=\"csi500\" will result in running the\ \ following configexamples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_csi500.yaml..\ \ code-block:: bash# Case 1 - run all models multiple timespython run_all_model.py\ \ run 3# Case 2 - run specific models multiple timespython run_all_model.py run\ \ 3 mlp# Case 3 - run specific models multiple times with specific datasetpython\ \ run_all_model.py run 3 mlp Alpha158# Case 4 - run other models except those\ \ are given as arguments for multiple timespython run_all_model.py run 3 [mlp,tft,lstm]\ \ --exclude=True# Case 5 - run specific models for one timepython run_all_model.py\ \ run --models=[mlp,lightgbm]# Case 6 - run other models except those are given\ \ as arguments for one timepython run_all_model.py run --models=[mlp,tft,sfm]\ \ --exclude=True# Case 7 - run lightgbm model on csi500.python run_all_model.py\ \ run 3 lightgbm Alpha158 csi500" function: run_all_model.run - docstring: null function: create_dataset.get_library_name - docstring: null function: create_dataset.is_stock - docstring: " \nexchange_place: \"SZ\" OR \"SH\"type: \"tick\", \"orderbook\"\ , ...filepath: the path of csvarc: arclink created by a process" function: create_dataset.add_one_stock_daily_data - docstring: null function: create_dataset.format_time - docstring: null function: create_dataset.add_one_stock_daily_data_wrapper - docstring: null function: create_dataset.add_data - docstring: ' Dataset creator ' function: 'create_dataset.DSCreator:' - docstring: null function: create_dataset.clear - docstring: null function: create_dataset.initialize_library - docstring: null function: create_dataset._get_empty_folder - docstring: " \nUseful commands- run all tests: pytest examples/orderbook_data/example.py-\ \ run a single test: pytest -s --pdb --disable-warnings examples/orderbook_data/example.py::TestClass::test_basic01" function: example.TestClass - docstring: " \nConfigure for arctic" function: example.setUp - docstring: null function: example.test_basic - docstring: null function: example.test_basic_without_time - docstring: null function: example.test_basic01 - docstring: null function: example.test_basic02 - docstring: null function: example.test_basic03 - docstring: null function: example.total_func - docstring: null function: example.test_exp_01 - docstring: null function: example.test_exp_02 - docstring: null function: example.test_exp_03 - docstring: null function: example.test_exp_04 - docstring: null function: example.test_exp_05 - docstring: null function: example.test_exp_06 - docstring: null function: example.expr7_init - docstring: null function: example.test_exp_07_1 - docstring: null function: example.test_exp_07_2 - docstring: null function: example.expr7_3_init - docstring: null function: example.test_exp_08_1 - docstring: null function: example.test_exp_08_2 - docstring: null function: example.test_exp_09_trans - docstring: null function: example.test_exp_09_order - docstring: null function: dataset._to_tensor - docstring: " \ncreate time series slices from pandas indexArgs:index (pd.MultiIndex):\ \ pandas multiindex with orderseq_len (int): sequence length" function: dataset._create_ts_slices - docstring: ' get date parse function This method is used to parse date arguments as target type.Example:get_date_parse_fn(''20120101'')(''2017-01-01'') => ''20170101''get_date_parse_fn(20120101)(''2017-01-01'') => 20170101' function: dataset._get_date_parse_fn - docstring: ' Memory Augmented Time Series Dataset Args:handler (DataHandler): data handlersegments (dict): data split segmentsseq_len (int): time series sequence lengthhorizon (int): label horizon (to mask historical loss for TRA)num_states (int): how many memory states to be added (for TRA)batch_size (int): batch size (<0 means daily batch)shuffle (bool): whether shuffle datapin_memory (bool): whether pin data to gpu memorydrop_last (bool): whether drop last batch < batch_size' function: dataset.MTSDatasetH - docstring: null function: dataset.setup_data - docstring: null function: dataset._prepare_seg - docstring: null function: dataset.restore_index - docstring: null function: dataset.assign_data - docstring: null function: dataset.clear_memory - docstring: ' enable traning mode self.batch_size, self.drop_last, self.shuffle = self.params' function: dataset.train - docstring: ' enable evaluation mode self.batch_size = -1self.drop_last = Falseself.shuffle = False' function: dataset.eval - docstring: null function: model.TRAModel - docstring: null function: model.train_epoch - docstring: null function: model.test_epoch - docstring: null function: model.fit - docstring: null function: model.predict - docstring: ' LSTM Model Args:input_size (int): input size (# features)hidden_size (int): hidden sizenum_layers (int): number of hidden layersuse_attn (bool): whether use attention layer.we use concat attention as https://github.com/fulifeng/Adv-ALSTM/dropout (float): dropout rateinput_drop (float): input dropout for data augmentationnoise_level (float): add gaussian noise to input for data augmentation' function: model.LSTM - docstring: null function: model.forward - docstring: null function: model.PositionalEncoding - docstring: null function: model.forward - docstring: ' Transformer Model Args:input_size (int): input size (# features)hidden_size (int): hidden sizenum_layers (int): number of transformer layersnum_heads (int): number of heads in transformerdropout (float): dropout rateinput_drop (float): input dropout for data augmentationnoise_level (float): add gaussian noise to input for data augmentation' function: model.Transformer - docstring: null function: model.forward - docstring: ' Temporal Routing Adaptor (TRA) TRA takes historical prediction errors & latent representation as inputs,then routes the input sample to a specific predictor for training & inference.Args:input_size (int): input size (RNN/Transformer''s hidden size)num_states (int): number of latent states (i.e., trading patterns)If `num_states=1`, then TRA falls back to traditional methodshidden_size (int): hidden size of the routertau (float): gumbel softmax temperature' function: model.TRA - docstring: null function: model.forward - docstring: null function: model.evaluate - docstring: null function: model.average_params - docstring: ' Replaces inf by maximum of tensor mask_inf = torch.isinf(inp_tensor)ind_inf = torch.nonzero(mask_inf, as_tuple=False)if len(ind_inf) > 0:for ind in ind_inf:if len(ind) == 2:inp_tensor[ind[0], ind[1]] = 0elif len(ind) == 1:inp_tensor[ind[0]] = 0m = torch.max(inp_tensor)for ind in ind_inf:if len(ind) == 2:inp_tensor[ind[0], ind[1]] = melif len(ind) == 1:inp_tensor[ind[0]] = mreturn inp_tensor' function: model.shoot_infs - docstring: null function: tft.get_shifted_label - docstring: null function: tft.fill_test_na - docstring: ' Prepare data to fit the TFT model. Args:df: Original DataFrame.fillna: Whether to fill the data with the mean values.Returns:Transformed DataFrame.' function: tft.process_qlib_data - docstring: ' Transform the TFT predicted data into Qlib format. Args:df: Original DataFrame.fillna: New column name.Returns:Transformed DataFrame.' function: tft.process_predicted - docstring: null function: tft.format_score - docstring: null function: tft.transform_df - docstring: ' TFT Model self.model = Noneself.params = {"DATASET": "Alpha158", "label_shift": 5}self.params.update(kwargs)' function: tft.TFTModel - docstring: null function: tft._prepare_data - docstring: null function: tft.fit - docstring: ' Strips out forecast time and identifier columns. return data[[col for col in data.columns if col not in {"forecast_time", "identifier"}]]# p50_loss = utils.numpy_normalised_quantile_loss(# extract_numerical_data(targets), extract_numerical_data(p50_forecast),# 0.5)# p90_loss = utils.numpy_normalised_quantile_loss(# extract_numerical_data(targets), extract_numerical_data(p90_forecast),# 0.9)tf.keras.backend.set_session(default_keras_session)print("Training completed at {}.".format(dte.datetime.now()))# ===========================Training Process===========================' function: tft.extract_numerical_data - docstring: null function: tft.predict - docstring: " \nfinetune modelParameters----------dataset : DatasetHdataset\ \ for finetuning" function: tft.finetune - docstring: " \nTensorflow model can't be dumped directly.So the data should\ \ be save separately**TODO**: Please implement the function to load the filesParameters----------path\ \ : Union[Path, str]the target path to be dumped" function: tft.to_pickle - docstring: ' Returns simple Keras linear layer. Args:size: Output sizeactivation: Activation function to apply if requireduse_time_distributed: Whether to apply layer across timeuse_bias: Whether bias should be included in layer' function: tft_model.linear_layer - docstring: ' Applies simple feed-forward network to an input. Args:inputs: MLP inputshidden_size: Hidden state sizeoutput_size: Output size of MLPoutput_activation: Activation function to apply on outputhidden_activation: Activation function to apply on inputuse_time_distributed: Whether to apply across timeReturns:Tensor for MLP outputs.' function: tft_model.apply_mlp - docstring: ' Applies a Gated Linear Unit (GLU) to an input. Args:x: Input to gating layerhidden_layer_size: Dimension of GLUdropout_rate: Dropout rate to apply if anyuse_time_distributed: Whether to apply across timeactivation: Activation function to apply to the linear feature transform ifnecessaryReturns:Tuple of tensors for: (GLU output, gate)' function: tft_model.apply_gating_layer - docstring: ' Applies skip connection followed by layer normalisation. Args:x_list: List of inputs to sum for skip connectionReturns:Tensor output from layer.' function: tft_model.add_and_norm - docstring: ' Applies the gated residual network (GRN) as defined in paper. Args:x: Network inputshidden_layer_size: Internal state sizeoutput_size: Size of output layerdropout_rate: Dropout rate if dropout is applieduse_time_distributed: Whether to apply network across time dimensionadditional_context: Additional context vector to use if relevantreturn_gate: Whether to return GLU gate for diagnostic purposesReturns:Tuple of tensors for: (GRN output, GLU gate)' function: tft_model.gated_residual_network - docstring: ' Returns causal mask to apply for self-attention layer. Args:self_attn_inputs: Inputs to self attention layer to determine mask shape' function: tft_model.get_decoder_mask - docstring: ' Defines scaled dot product attention layer. Attributes:dropout: Dropout rate to useactivation: Normalisation function for scaled dot product attention (e.g.softmax by default)' function: 'tft_model.ScaledDotProductAttention:' - docstring: ' Defines interpretable multi-head attention layer. Attributes:n_head: Number of headsd_k: Key/query dimensionality per headd_v: Value dimensionalitydropout: Dropout rate to applyqs_layers: List of queries across headsks_layers: List of keys across headsvs_layers: List of values across headsattention: Scaled dot product attention layerw_o: Output weight matrix to project internal state to the original TFTstate size' function: 'tft_model.InterpretableMultiHeadAttention:' - docstring: ' Caches data for the TFT. _data_cache = {}@classmethod' function: 'tft_model.TFTDataCache:' - docstring: ' Updates cached data. Args:data: Source to updatekey: Key to dictionary location' function: tft_model.update - docstring: ' Returns data stored at key location. return cls._data_cache[key].copy()@classmethod' function: tft_model.get - docstring: ' Returns boolean indicating whether key is present in cache. return key in cls._data_cache# TFT model definitions.' function: tft_model.contains - docstring: ' Defines Temporal Fusion Transformer. Attributes:name: Name of modeltime_steps: Total number of input time steps per forecast date (i.e. Widthof Temporal fusion decoder N)input_size: Total number of inputsoutput_size: Total number of outputscategory_counts: Number of categories per categorical variablen_multiprocessing_workers: Number of workers to use for parallelcomputationscolumn_definition: List of tuples of (string, DataType, InputType) thatdefine each columnquantiles: Quantiles to forecast for TFTuse_cudnn: Whether to use Keras CuDNNLSTM or standard LSTM layershidden_layer_size: Internal state size of TFTdropout_rate: Dropout discard ratemax_gradient_norm: Maximum norm for gradient clippinglearning_rate: Initial learning rate of ADAM optimizerminibatch_size: Size of minibatches for trainingnum_epochs: Maximum number of epochs for trainingearly_stopping_patience: Maximum number of iterations of non-improvementbefore early stopping kicks innum_encoder_steps: Size of LSTM encoder -- i.e. number of past time stepsbefore forecast date to usenum_stacks: Number of self-attention layers to apply (default is 1 for basicTFT)num_heads: Number of heads for interpretable mulit-head attentionmodel: Keras model for TFT' function: 'tft_model.TemporalFusionTransformer:' - docstring: ' Transforms raw inputs to embeddings. Applies linear transformation onto continuous variables and uses embeddingsfor categorical variables.Args:all_inputs: Inputs to transformReturns:Tensors for transformed inputs.' function: tft_model.get_tft_embeddings - docstring: ' Applies linear transformation for time-varying inputs. return tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(self.hidden_layer_size))(x)# Targetsobs_inputs = tf.keras.backend.stack([convert_real_to_embedding(regular_inputs[Ellipsis, i : i + 1]) for i in self._input_obs_loc], axis=-1)# Observed (a prioir unknown) inputswired_embeddings = []for i in range(num_categorical_variables):if i not in self._known_categorical_input_idx and i + num_regular_variables not in self._input_obs_loc:e = embeddings[i](categorical_inputs[:, :, i])wired_embeddings.append(e)unknown_inputs = []for i in range(regular_inputs.shape[-1]):if i not in self._known_regular_input_idx and i not in self._input_obs_loc:e = convert_real_to_embedding(regular_inputs[Ellipsis, i : i + 1])unknown_inputs.append(e)if unknown_inputs + wired_embeddings:unknown_inputs = tf.keras.backend.stack(unknown_inputs + wired_embeddings, axis=-1)else:unknown_inputs = None# A priori known inputsknown_regular_inputs = [convert_real_to_embedding(regular_inputs[Ellipsis, i : i + 1])for i in self._known_regular_input_idxif i not in self._static_input_loc]known_categorical_inputs = [embedded_inputs[i]for i in self._known_categorical_input_idxif i + num_regular_variables not in self._static_input_loc]known_combined_layer = tf.keras.backend.stack(known_regular_inputs + known_categorical_inputs, axis=-1)return unknown_inputs, known_combined_layer, obs_inputs, static_inputs' function: tft_model.convert_real_to_embedding - docstring: ' Returns name of single column for input type. return utils.get_single_col_by_input_type(input_type, self.column_definition)' function: tft_model._get_single_col_by_type - docstring: ' Returns boolean indicating if training data has been cached. return TFTDataCache.contains("train") and TFTDataCache.contains("valid")' function: tft_model.training_data_cached - docstring: ' Batches and caches data once for using during training. Args:data: Data to batch and cachecache_key: Key used for cachenum_samples: Maximum number of samples to extract (-1 to use all data)' function: tft_model.cache_batched_data - docstring: ' Samples segments into a compatible format. Args:data: Sources data to sample and batchmax_samples: Maximum number of samples in batchReturns:Dictionary of batched data with the maximum samples specified.' function: tft_model._batch_sampled_data - docstring: ' Batches data for training. Converts raw dataframe from a 2-D tabular format to a batched 3-D arrayto feed into Keras model.Args:data: DataFrame to batchReturns:Batched Numpy array with shape=(?, self.time_steps, self.input_size)' function: tft_model._batch_data - docstring: null function: tft_model._batch_single_entity - docstring: ' Formats sample weights for Keras training. return (np.sum(x, axis=-1) > 0.0) * 1.0' function: tft_model._get_active_locations - docstring: ' Returns graph defining layers of the TFT. # Size definitions.time_steps = self.time_stepscombined_input_size = self.input_sizeencoder_steps = self.num_encoder_steps# Inputs.all_inputs = tf.keras.layers.Input(shape=(time_steps,combined_input_size,))unknown_inputs, known_combined_layer, obs_inputs, static_inputs = self.get_tft_embeddings(all_inputs)# Isolate known and observed historical inputs.if unknown_inputs is not None:historical_inputs = concat([unknown_inputs[:, :encoder_steps, :],known_combined_layer[:, :encoder_steps, :],obs_inputs[:, :encoder_steps, :],],axis=-1,)else:historical_inputs = concat([known_combined_layer[:, :encoder_steps, :], obs_inputs[:, :encoder_steps, :]], axis=-1)# Isolate only known future inputs.future_inputs = known_combined_layer[:, encoder_steps:, :]' function: tft_model._build_base_graph - docstring: ' Applies variable selection network to static inputs. Args:embedding: Transformed static inputsReturns:Tensor output for variable selection network' function: tft_model.static_combine_and_mask - docstring: ' Apply temporal variable selection networks. Args:embedding: Transformed inputs.Returns:Processed tensor outputs.' function: tft_model.lstm_combine_and_mask - docstring: ' Returns LSTM cell initialized with default parameters. if self.use_cudnn:lstm = tf.keras.layers.CuDNNLSTM(self.hidden_layer_size,return_sequences=True,return_state=return_state,stateful=False,)else:lstm = tf.keras.layers.LSTM(self.hidden_layer_size,return_sequences=True,return_state=return_state,stateful=False,# Additional params to ensure LSTM matches CuDNN, See TF 2.0 :# (https://www.tensorflow.org/api_docs/python/tf/keras/layers/LSTM)activation="tanh",recurrent_activation="sigmoid",recurrent_dropout=0,unroll=False,use_bias=True,)return lstmhistory_lstm, state_h, state_c = get_lstm(return_state=True)(historical_features, initial_state=[static_context_state_h, static_context_state_c])future_lstm = get_lstm(return_state=False)(future_features, initial_state=[state_h, state_c])lstm_layer = concat([history_lstm, future_lstm], axis=1)# Apply gated skip connectioninput_embeddings = concat([historical_features, future_features], axis=1)lstm_layer, _ = apply_gating_layer(lstm_layer, self.hidden_layer_size, self.dropout_rate, activation=None)temporal_feature_layer = add_and_norm([lstm_layer, input_embeddings])# Static enrichment layersexpanded_static_context = K.expand_dims(static_context_enrichment, axis=1)enriched, _ = gated_residual_network(temporal_feature_layer,self.hidden_layer_size,dropout_rate=self.dropout_rate,use_time_distributed=True,additional_context=expanded_static_context,return_gate=True,)# Decoder self attentionself_attn_layer = InterpretableMultiHeadAttention(self.num_heads, self.hidden_layer_size, dropout=self.dropout_rate)mask = get_decoder_mask(enriched)x, self_att = self_attn_layer(enriched, enriched, enriched, mask=mask)x, _ = apply_gating_layer(x, self.hidden_layer_size, dropout_rate=self.dropout_rate, activation=None)x = add_and_norm([x, enriched])# Nonlinear processing on outputsdecoder = gated_residual_network(x, self.hidden_layer_size, dropout_rate=self.dropout_rate, use_time_distributed=True)# Final skip connectiondecoder, _ = apply_gating_layer(decoder, self.hidden_layer_size, activation=None)transformer_layer = add_and_norm([decoder, temporal_feature_layer])# Attention components for explainabilityattention_components = {# Temporal attention weights"decoder_self_attn": self_att,# Static variable selection weights"static_flags": static_weights[Ellipsis, 0],# Variable selection weights of past inputs"historical_flags": historical_flags[Ellipsis, 0, :],# Variable selection weights of future inputs"future_flags": future_flags[Ellipsis, 0, :],}return transformer_layer, all_inputs, attention_components' function: tft_model.get_lstm - docstring: ' Build model and defines training losses. Returns:Fully defined Keras model.' function: tft_model.build_model - docstring: ' Computes the combined quantile loss for prespecified quantiles. Attributes:quantiles: Quantiles to compute losses' function: 'tft_model.QuantileLossCalculator:' - docstring: ' Returns quantile loss for specified quantiles. Args:a: Targetsb: Predictions' function: tft_model.quantile_loss - docstring: ' Fits deep neural network for given training and validation data. Args:train_df: DataFrame for training datavalid_df: DataFrame for validation data' function: tft_model.fit - docstring: null function: tft_model._unpack - docstring: ' Applies evaluation metric to the training data. Args:data: Dataframe for evaluationeval_metric: Evaluation metic to return, based on model definition.Returns:Computed evaluation loss.' function: tft_model.evaluate - docstring: ' Computes predictions for a given input dataset. Args:df: Input dataframereturn_targets: Whether to also return outputs aligned with predictions tofacilitate evaluationReturns:Input dataframe or tuple of (input dataframe, aligned output dataframe).' function: tft_model.predict - docstring: ' Returns formatted dataframes for prediction. flat_prediction = pd.DataFrame(prediction[:, :, 0], columns=["t+{}".format(i) for i in range(self.time_steps - self.num_encoder_steps)])cols = list(flat_prediction.columns)flat_prediction["forecast_time"] = time[:, self.num_encoder_steps - 1, 0]flat_prediction["identifier"] = identifier[:, 0, 0]# Arrange in orderreturn flat_prediction[["forecast_time", "identifier"] + cols]# Extract predictions for each quantile into different entriesprocess_map = {"p{}".format(int(q * 100)): combined[Ellipsis, i * self.output_size : (i + 1) * self.output_size]for i, q in enumerate(self.quantiles)}if return_targets:# Add targets if relevantprocess_map["targets"] = outputsreturn {k: format_outputs(process_map[k]) for k in process_map}' function: tft_model.format_outputs - docstring: ' Computes TFT attention weights for a given dataset. Args:df: Input dataframeReturns:Dictionary of numpy arrays for temporal attention weights and variableselection weights, along with their identifiers and time indices' function: tft_model.get_attention - docstring: ' Returns weights for a given minibatch of data. input_placeholder = self._input_placeholderattention_weights = {}for k in self._attention_components:attention_weight = tf.keras.backend.get_session().run(self._attention_components[k], {input_placeholder: input_batch.astype(np.float32)})attention_weights[k] = attention_weightreturn attention_weights# Compute number of batchesbatch_size = self.minibatch_sizen = inputs.shape[0]num_batches = n // batch_sizeif n - (num_batches * batch_size) > 0:num_batches += 1# Split up inputs into batchesbatched_inputs = [inputs[i * batch_size : (i + 1) * batch_size, Ellipsis] for i in range(num_batches)]# Get attention weights, while avoiding large memory increasesattention_by_batch = [get_batch_attention_weights(batch) for batch in batched_inputs]attention_weights = {}for k in self._attention_components:attention_weights[k] = []for batch_weights in attention_by_batch:attention_weights[k].append(batch_weights[k])if len(attention_weights[k][0].shape) == 4:tmp = np.concatenate(attention_weights[k], axis=1)else:tmp = np.concatenate(attention_weights[k], axis=0)del attention_weights[k]gc.collect()attention_weights[k] = tmpattention_weights["identifiers"] = identifiers[:, 0, 0]attention_weights["time"] = time[:, :, 0]return attention_weights# Serialisation.' function: tft_model.get_batch_attention_weights - docstring: ' Deletes and recreates folder with temporary Keras training outputs. print("Resetting temp folder...")utils.create_folder_if_not_exist(self._temp_folder)shutil.rmtree(self._temp_folder)os.makedirs(self._temp_folder)' function: tft_model.reset_temp_folder - docstring: ' Returns path to keras checkpoint. return os.path.join(model_folder, "{}.check".format(self.name))' function: tft_model.get_keras_saved_path - docstring: ' Saves optimal TFT weights. Args:model_folder: Location to serialze model.' function: tft_model.save - docstring: ' Loads TFT weights. Args:model_folder: Folder containing serialized models.use_keras_loadings: Whether to load from Keras checkpoint.Returns:' function: tft_model.load - docstring: ' Returns name of single column. Args:input_type: Input type of column to extractcolumn_definition: Column definition list for experiment' function: utils.get_single_col_by_input_type - docstring: ' Extracts the names of columns that correspond to a define data_type. Args:data_type: DataType of columns to extract.column_definition: Column definition to use.excluded_input_types: Set of input types to excludeReturns:List of names for columns with data type specified.' function: utils.extract_cols_from_data_type - docstring: ' Computes quantile loss for tensorflow. Standard quantile loss as defined in the "Training Procedure" section ofthe main TFT paperArgs:y: Targetsy_pred: Predictionsquantile: Quantile to use for loss calculations (between 0 & 1)Returns:Tensor for quantile loss.' function: utils.tensorflow_quantile_loss - docstring: ' Computes normalised quantile loss for numpy arrays. Uses the q-Risk metric as defined in the "Training Procedure" section of themain TFT paper.Args:y: Targetsy_pred: Predictionsquantile: Quantile to use for loss calculations (between 0 & 1)Returns:Float for normalised quantile loss.' function: utils.numpy_normalised_quantile_loss - docstring: ' Creates folder if it doesn''t exist. Args:directory: Folder path to create.' function: utils.create_folder_if_not_exist - docstring: ' Creates tensorflow config for graphs to run on CPU or GPU. Specifies whether to run graph on gpu or cpu and which GPU ID to use for multiGPU machines.Args:tf_device: ''cpu'' or ''gpu''gpu_id: GPU ID to use if relevantReturns:Tensorflow config.' function: utils.get_default_tensorflow_config - docstring: ' Saves Tensorflow graph to checkpoint. Saves all trainiable variables under a given variable scope to checkpoint.Args:tf_session: Session containing graphmodel_folder: Folder to save modelscp_name: Name of Tensorflow checkpointscope: Variable scope containing variables to save' function: utils.save - docstring: ' Loads Tensorflow graph from checkpoint. Args:tf_session: Session to load graph intomodel_folder: Folder containing serialised modelcp_name: Name of Tensorflow checkpointscope: Variable scope to use.verbose: Whether to print additional debugging information.' function: utils.load - docstring: ' Prints all weights in Tensorflow checkpoint. Args:model_folder: Folder containing checkpointcp_name: Name of checkpointReturns:' function: utils.print_weights_in_checkpoint - docstring: ' Manages hyperparameter optimisation using random search for a single GPU. Attributes:param_ranges: Discrete hyperparameter range for random search.results: Dataframe of validation results.fixed_params: Fixed model parameters per experiment.saved_params: Dataframe of parameters trained.best_score: Minimum validation loss observed thus far.optimal_name: Key to best configuration.hyperparam_folder: Where to save optimisation outputs.' function: 'hyperparam_opt.HyperparamOptManager:' - docstring: ' Loads results from previous hyperparameter optimisation. Returns:A boolean indicating if previous results can be loaded.' function: hyperparam_opt.load_results - docstring: ' Returns previously saved parameters given a key. params = self.saved_paramsselected_params = dict(params[name])if self._override_w_fixed_params:for k in self.fixed_params:selected_params[k] = self.fixed_params[k]return selected_params' function: hyperparam_opt._get_params_from_name - docstring: ' Returns the optimal hyperparameters thus far. optimal_name = self.optimal_namereturn self._get_params_from_name(optimal_name)' function: hyperparam_opt.get_best_params - docstring: ' Clears all previous results and saved parameters. shutil.rmtree(self.hyperparam_folder)os.makedirs(self.hyperparam_folder)self.results = pd.DataFrame()self.saved_params = pd.DataFrame()' function: hyperparam_opt.clear - docstring: ' Checks that parameter map is properly defined. valid_fields = list(self.param_ranges.keys()) + list(self.fixed_params.keys())invalid_fields = [k for k in params if k not in valid_fields]missing_fields = [k for k in valid_fields if k not in params]if invalid_fields:raise ValueError("Invalid Fields Found {} - Valid ones are {}".format(invalid_fields, valid_fields))if missing_fields:raise ValueError("Missing Fields Found {} - Valid ones are {}".format(missing_fields, valid_fields))' function: hyperparam_opt._check_params - docstring: ' Returns a unique key for the supplied set of params. self._check_params(params)fields = list(params.keys())fields.sort()return "_".join([str(params[k]) for k in fields])' function: hyperparam_opt._get_name - docstring: ' Returns the next set of parameters to optimise. Args:ranges_to_skip: Explicitly defines a set of keys to skip.' function: hyperparam_opt.get_next_parameters - docstring: ' Returns next hyperparameter set per try. parameters = {k: np.random.choice(self.param_ranges[k]) for k in param_range_keys}# Adds fixed paramsfor k in self.fixed_params:parameters[k] = self.fixed_params[k]return parametersfor _ in range(self._max_tries):parameters = _get_next()name = self._get_name(parameters)if name not in ranges_to_skip:return parametersraise ValueError("Exceeded max number of hyperparameter searches!!")' function: hyperparam_opt._get_next - docstring: ' Updates the results from last optimisation run. Args:parameters: Hyperparameters used in optimisation.loss: Validation loss obtained.model: Model to serialised if required.info: Any ancillary information to tag on to results.Returns:Boolean flag indicating if the model is the best seen so far.' function: hyperparam_opt.update_score - docstring: ' Manages distributed hyperparameter optimisation across many gpus. self,param_ranges,fixed_params,root_model_folder,worker_number,search_iterations=1000,num_iterations_per_worker=5,clear_serialised_params=False,):Instantiates optimisation manager.This hyperparameter optimisation pre-generates #search_iterationshyperparameter combinations and serialises themat the start. At runtime, each worker goes through their own set ofparameter ranges. The pregenerationallows for multiple workers to run in parallel on different machines withoutresulting in parameter overlaps.Args:param_ranges: Discrete hyperparameter range for random search.fixed_params: Fixed model parameters per experiment.root_model_folder: Folder to store optimisation artifacts.worker_number: Worker index defining which set of hyperparameters totest.search_iterations: Maximum number of random search iterations.num_iterations_per_worker: How many iterations are handled per worker.clear_serialised_params: Whether to regenerate hyperparametercombinations.' function: hyperparam_opt.DistributedHyperparamOptManager - docstring: null function: hyperparam_opt.optimisation_completed - docstring: ' Returns next dictionary of hyperparameters to optimise. param_name = self.worker_search_queue.pop()params = self.global_hyperparam_df.loc[param_name, :].to_dict()# Always override!for k in self.fixed_params:print("Overriding saved {}: {}".format(k, self.fixed_params[k]))params[k] = self.fixed_params[k]return params' function: hyperparam_opt.get_next_parameters - docstring: ' Loads serialsed hyperparameter ranges from file. Returns:DataFrame containing hyperparameter combinations.' function: hyperparam_opt.load_serialised_hyperparam_df - docstring: ' Regenerates hyperparameter combinations and saves to file. Returns:DataFrame containing hyperparameter combinations.' function: hyperparam_opt.update_serialised_hyperparam_df - docstring: ' Generates actual hyperparameter combinations. Returns:DataFrame containing hyperparameter combinations.' function: hyperparam_opt._generate_full_hyperparam_df - docstring: ' Clears results for hyperparameter manager and resets. super().clear()self.worker_search_queue = self._get_worker_search_queue()' function: hyperparam_opt.clear - docstring: ' Load results from file and queue parameter combinations to try. Returns:Boolean indicating if results were successfully loaded.' function: hyperparam_opt.load_results - docstring: ' Generates the queue of param combinations for current worker. Returns:Queue of hyperparameter combinations outstanding.' function: hyperparam_opt._get_worker_search_queue - docstring: ' Updates parameter combinations with the index of the worker used. Args:df: DataFrame of parameter combinations.Returns:Updated DataFrame with worker number.' function: hyperparam_opt.assign_worker_numbers - docstring: ' Defines experiment configs and paths to outputs. Attributes:root_folder: Root folder to contain all experimental outputs.experiment: Name of experiment to run.data_folder: Folder to store data for experiment.model_folder: Folder to store serialised models.results_folder: Folder to store results.data_csv_path: Path to primary data csv file used in experiment.hyperparam_iterations: Default number of random search iterations forexperiment.' function: 'configs.ExperimentConfig:' - docstring: null function: configs.data_csv_path - docstring: null function: configs.hyperparam_iterations - docstring: ' Gets a data formatter object for experiment. Returns:Default DataFormatter per experiment.' function: configs.make_data_formatter - docstring: ' Defines and formats data for the Alpha158 dataset. Attributes:column_definition: Defines input and data type of column used in theexperiment.identifiers: Entity identifiers used in experiments.' function: qlib_Alpha158.Alpha158Formatter - docstring: ' Splits data frame into training-validation-test data frames. This also calibrates scaling object, and transforms data for each split.Args:df: Source data frame to split.valid_boundary: Starting year for validation datatest_boundary: Starting year for test dataReturns:Tuple of transformed (train, valid, test) data.' function: qlib_Alpha158.split_data - docstring: ' Calibrates scalers using the data supplied. Args:df: Data to use to calibrate scalers.' function: qlib_Alpha158.set_scalers - docstring: ' Performs feature transformations. This includes both feature engineering, preprocessing and normalisation.Args:df: Data frame to transform.Returns:Transformed data frame.' function: qlib_Alpha158.transform_inputs - docstring: ' Reverts any normalisation to give predictions in original scale. Args:predictions: Dataframe of model predictions.Returns:Data frame of unnormalised predictions.' function: qlib_Alpha158.format_predictions - docstring: ' Returns fixed model parameters for experiments. fixed_params = {"total_time_steps": 6 + 6,"num_encoder_steps": 6,"num_epochs": 100,"early_stopping_patience": 10,"multiprocessing_workers": 5,}return fixed_params' function: qlib_Alpha158.get_fixed_params - docstring: ' Defines numerical types of each column. REAL_VALUED = 0CATEGORICAL = 1DATE = 2' function: base.DataTypes - docstring: ' Defines input types of each column. TARGET = 0OBSERVED_INPUT = 1KNOWN_INPUT = 2STATIC_INPUT = 3ID = 4 # Single column used as an entity identifierTIME = 5 # Single column exclusively used as a time index' function: base.InputTypes - docstring: ' Abstract base class for all data formatters. User can implement the abstract methods below to perform dataset-specificmanipulations.' function: base.GenericDataFormatter - docstring: ' Calibrates scalers using the data supplied. raise NotImplementedError()@abc.abstractmethod' function: base.set_scalers - docstring: ' Performs feature transformation. raise NotImplementedError()@abc.abstractmethod' function: base.transform_inputs - docstring: ' Reverts any normalisation to give predictions in original scale. raise NotImplementedError()@abc.abstractmethod' function: base.format_predictions - docstring: ' Performs the default train, validation and test splits. raise NotImplementedError()@property@abc.abstractmethod' function: base.split_data - docstring: ' Defines order, input type and data type of each column. raise NotImplementedError()@abc.abstractmethod' function: base._column_definition - docstring: ' Defines the fixed parameters used by the model for training. Requires the following keys:''total_time_steps'': Defines the total number of time steps used by TFT''num_encoder_steps'': Determines length of LSTM encoder (i.e. history)''num_epochs'': Maximum number of epochs for training''early_stopping_patience'': Early stopping param for keras''multiprocessing_workers'': # of cpus for data processingReturns:A dictionary of fixed parameters, e.g.:fixed_params = {''total_time_steps'': 252 + 5,''num_encoder_steps'': 252,''num_epochs'': 100,''early_stopping_patience'': 5,''multiprocessing_workers'': 5,}' function: base.get_fixed_params - docstring: ' Returns number of categories per relevant input. This is seqeuently required for keras embedding layers.' function: base.num_classes_per_cat_input - docstring: ' Gets the default number of training and validation samples. Use to sub-sample the data for network calibration and a value of -1 usesall available samples.Returns:Tuple of (training samples, validation samples)' function: base.get_num_samples_for_calibration - docstring: ' Returns formatted column definition in order expected by the TFT. column_definition = self._column_definition# Sanity checks first.# Ensure only one ID and time column exist' function: base.get_column_definition - docstring: null function: base._check_single_column - docstring: ' Returns names of all input columns. return [tup[0] for tup in self.get_column_definition() if tup[2] not in {InputTypes.ID, InputTypes.TIME}]' function: base._get_input_columns - docstring: ' Returns the relevant indexes and input sizes required by TFT. # Functions' function: base._get_tft_input_indices - docstring: null function: base._extract_tuples_from_data_type - docstring: null function: base._get_locations - docstring: null function: multi_freq_handler.Avg15minLoader - docstring: null function: multi_freq_handler.load - docstring: null function: multi_freq_handler.Avg15minHandler - docstring: null function: 'workflow.NestedDecisionExecutionWorkflow:' - docstring: ' initialize qlib provider_uri_day = "~/.qlib/qlib_data/cn_data" # target_dirGetData().qlib_data(target_dir=provider_uri_day, region=REG_CN, version="v2", exists_skip=True)provider_uri_1min = HIGH_FREQ_CONFIG.get("provider_uri")GetData().qlib_data(target_dir=provider_uri_1min, interval="1min", region=REG_CN, version="v2", exists_skip=True)provider_uri_map = {"1min": provider_uri_1min, "day": provider_uri_day}qlib.init(provider_uri=provider_uri_map, dataset_cache=None, expression_cache=None)' function: workflow._init_qlib - docstring: null function: workflow._train_model - docstring: null function: workflow.backtest - docstring: null function: workflow.collect_data - docstring: null function: workflow.check_diff_freq - docstring: " \nThis backtest is used for comparing the nested execution and\ \ single layer executionDue to the low quality daily-level and miniute-level data,\ \ they are hardly comparable.So it is used for detecting serious bugs which make\ \ the results different greatly... code-block:: shell[1724971:MainThread](2021-12-07\ \ 16:24:31,156) INFO - qlib.workflow - [record_temp.py:441] - Portfolio analysis\ \ record 'port_analysis_1day.pkl'has been saved as the artifact of the Experiment\ \ 2'The following are analysis results of benchmark return(1day).'riskmean \ \ 0.000651std 0.012472annualized_return 0.154967information_ratio\ \ 0.805422max_drawdown -0.160445'The following are analysis results of the\ \ excess return without cost(1day).'riskmean 0.001375std \ \ 0.006103annualized_return 0.327204information_ratio 3.475016max_drawdown\ \ -0.024927'The following are analysis results of the excess return with\ \ cost(1day).'riskmean 0.001184std 0.006091annualized_return\ \ 0.281801information_ratio 2.998749max_drawdown -0.029568[1724971:MainThread](2021-12-07\ \ 16:24:31,170) INFO - qlib.workflow - [record_temp.py:466] - Indicator analysis\ \ record 'indicator_analysis_1day.pkl' has been saved as the artifact of the Experiment\ \ 2'The following are analysis results of indicators(1day).'valueffr 1.0pa\ \ 0.0pos 0.0[1724971:MainThread](2021-12-07 16:24:31,188) INFO - qlib.timer\ \ - [log.py:113] - Time cost: 0.007s | waiting `async_log` Done" function: workflow.backtest_only_daily - docstring: null function: 'task_manager_rolling.RollingTaskExample:' - docstring: null function: task_manager_rolling.reset - docstring: null function: task_manager_rolling.task_generating - docstring: null function: task_manager_rolling.task_training - docstring: null function: task_manager_rolling.worker - docstring: null function: task_manager_rolling.task_collecting - docstring: null function: task_manager_rolling.rec_key - docstring: null function: task_manager_rolling.my_filter - docstring: ' DayLast Operator Parameters----------feature : Expressionfeature instanceReturns----------feature:a series of that each value equals the last value of its day' function: highfreq_ops.DayLast - docstring: null function: highfreq_ops._load_internal - docstring: ' FFillNan Operator Parameters----------feature : Expressionfeature instanceReturns----------feature:a forward fill nan feature' function: highfreq_ops.FFillNan - docstring: null function: highfreq_ops._load_internal - docstring: ' BFillNan Operator Parameters----------feature : Expressionfeature instanceReturns----------feature:a backfoward fill nan feature' function: highfreq_ops.BFillNan - docstring: null function: highfreq_ops._load_internal - docstring: ' Date Operator Parameters----------feature : Expressionfeature instanceReturns----------feature:a series of that each value is the date corresponding to feature.index' function: highfreq_ops.Date - docstring: null function: highfreq_ops._load_internal - docstring: ' Select Operator Parameters----------feature_left : Expressionfeature instance, select conditionfeature_right : Expressionfeature instance, select valueReturns----------feature:value(feature_right) that meets the condition(feature_left)' function: highfreq_ops.Select - docstring: null function: highfreq_ops._load_internal - docstring: ' IsNull Operator Parameters----------feature : Expressionfeature instanceReturns----------feature:A series indicating whether the feature is nan' function: highfreq_ops.IsNull - docstring: null function: highfreq_ops._load_internal - docstring: ' Cut Operator Parameters----------feature : Expressionfeature instancel : intl > 0, delete the first l elements of feature (default is None, which means 0)r : intr < 0, delete the last -r elements of feature (default is None, which means 0)Returns----------feature:A series with the first l and last -r elements deleted from the feature.Note: It is deleted from the raw data, not the sliced data' function: highfreq_ops.Cut - docstring: null function: highfreq_ops._load_internal - docstring: null function: 'workflow.HighfreqWorkflow:' - docstring: ' initialize qlib # use cn_data_1min dataQLIB_INIT_CONFIG = {**HIGH_FREQ_CONFIG, **self.SPEC_CONF}provider_uri = QLIB_INIT_CONFIG.get("provider_uri")GetData().qlib_data(target_dir=provider_uri, interval="1min", region=REG_CN, exists_skip=True)qlib.init(**QLIB_INIT_CONFIG)' function: workflow._init_qlib - docstring: ' preload the calendar for cache # This code used the copy-on-write feature of Linux to avoid calculating the calendar multiple times in the subprocess# This code may accelerate, but may be not useful on Windows and Mac OsCal.calendar(freq="1min")get_calendar_day(freq="1min")' function: workflow._prepare_calender_cache - docstring: ' use dataset to get highreq data self._init_qlib()self._prepare_calender_cache()dataset = init_instance_by_config(self.task["dataset"])xtrain, xtest = dataset.prepare(["train", "test"])print(xtrain, xtest)dataset_backtest = init_instance_by_config(self.task["dataset_backtest"])backtest_train, backtest_test = dataset_backtest.prepare(["train", "test"])print(backtest_train, backtest_test)return' function: workflow.get_data - docstring: null function: highfreq_handler.HighFreqHandler - docstring: null function: highfreq_handler.get_feature_config - docstring: ' Get normalized price feature ops if shift == 0:template_norm = "Cut({0}/Ref(DayLast({1}), 240), 240, None)"else:template_norm = "Cut(Ref({0}, " + str(shift) + ")/Ref(DayLast({1}), 240), 240, None)"feature_ops = template_norm.format(template_if.format(template_fillnan.format(template_paused.format("$close")),template_paused.format(price_field),),template_fillnan.format(template_paused.format("$close")),)return feature_opsfields += [get_normalized_price_feature("$open", 0)]fields += [get_normalized_price_feature("$high", 0)]fields += [get_normalized_price_feature("$low", 0)]fields += [get_normalized_price_feature("$close", 0)]fields += [get_normalized_price_feature(simpson_vwap, 0)]names += ["$open", "$high", "$low", "$close", "$vwap"]fields += [get_normalized_price_feature("$open", 240)]fields += [get_normalized_price_feature("$high", 240)]fields += [get_normalized_price_feature("$low", 240)]fields += [get_normalized_price_feature("$close", 240)]fields += [get_normalized_price_feature(simpson_vwap, 240)]names += ["$open_1", "$high_1", "$low_1", "$close_1", "$vwap_1"]fields += ["Cut({0}/Ref(DayLast(Mean({0}, 7200)), 240), 240, None)".format("If(IsNull({0}), 0, If(Or(Gt({1}, Mul(1.001, {3})), Lt({1}, Mul(0.999, {2}))), 0, {0}))".format(template_paused.format("$volume"),template_paused.format(simpson_vwap),template_paused.format("$low"),template_paused.format("$high"),))]names += ["$volume"]fields += ["Cut(Ref({0}, 240)/Ref(DayLast(Mean({0}, 7200)), 240), 240, None)".format("If(IsNull({0}), 0, If(Or(Gt({1}, Mul(1.001, {3})), Lt({1}, Mul(0.999, {2}))), 0, {0}))".format(template_paused.format("$volume"),template_paused.format(simpson_vwap),template_paused.format("$low"),template_paused.format("$high"),))]names += ["$volume_1"]return fields, names' function: highfreq_handler.get_normalized_price_feature - docstring: null function: highfreq_handler.HighFreqBacktestHandler - docstring: null function: highfreq_processor.HighFreqNorm - docstring: null function: 'update_online_pred.UpdatePredExample:' - docstring: null function: update_online_pred.first_train - docstring: null function: update_online_pred.update_online_pred - docstring: null function: 'rolling_online_management.RollingOnlineExample:' - docstring: null function: rolling_online_management.worker - docstring: null function: rolling_online_management.reset - docstring: null function: rolling_online_management.first_run - docstring: null function: rolling_online_management.routine - docstring: null function: rolling_online_management.add_strategy - docstring: " \nInit OnlineManagerExample.Args:provider_uri (str, optional):\ \ the provider uri. Defaults to \"~/.qlib/qlib_data/cn_data\".region (str, optional):\ \ the stock region. Defaults to \"cn\".exp_name (str, optional): the experiment\ \ name. Defaults to \"rolling_exp\".task_url (str, optional): your MongoDB url.\ \ Defaults to \"mongodb://10.0.0.4:27017/\".task_db_name (str, optional): database\ \ name. Defaults to \"rolling_db\".task_pool (str, optional): the task pool name\ \ (a task pool is a collection in MongoDB). Defaults to \"rolling_task\".rolling_step\ \ (int, optional): the step for rolling. Defaults to 80.start_time (str, optional):\ \ the start time of simulating. Defaults to \"2018-09-10\".end_time (str, optional):\ \ the end time of simulating. Defaults to \"2018-10-31\".tasks (dict or list[dict]):\ \ a set of the task config waiting for rolling and training" function: 'online_management_simulate.OnlineSimulationExample:' - docstring: null function: online_management_simulate.reset - docstring: null function: online_management_simulate.main - docstring: null function: 'workflow.RollingDataWorkflow:' - docstring: ' initialize qlib provider_uri = "~/.qlib/qlib_data/cn_data" # target_dirGetData().qlib_data(target_dir=provider_uri, region=REG_CN, exists_skip=True)qlib.init(provider_uri=provider_uri, region=REG_CN)' function: workflow._init_qlib - docstring: null function: workflow._dump_pre_handler - docstring: null function: workflow._load_pre_handler - docstring: " \nUser could collect system info by following commands`cd scripts\ \ && python collect_info.py all`- NOTE: please avoid running this script in the\ \ project folder which contains `qlib`" function: 'collect_info.InfoCollector:' - docstring: ' collect system related info for method in ["system", "machine", "platform", "version"]:print(getattr(platform, method)())' function: collect_info.sys - docstring: ' collect Python related info print("Python version: {}".format(sys.version.replace("\n", " ")))' function: collect_info.py - docstring: ' collect qlib related info print("Qlib version: {}".format(qlib.__version__))REQUIRED = ["numpy","pandas","scipy","requests","sacred","python-socketio","redis","python-redis-lock","schedule","cvxpy","hyperopt","fire","statsmodels","xlrd","plotly","matplotlib","tables","pyyaml","mlflow","tqdm","loguru","lightgbm","tornado","joblib","fire","ruamel.yaml",]for package in REQUIRED:version = pkg_resources.get_distribution(package).versionprint(f"{package}=={version}")' function: collect_info.qlib - docstring: " \nParameters----------csv_path: strstock data path or directoryqlib_dir:\ \ strqlib(dump) data directorbackup_dir: str, default Noneif backup_dir is not\ \ None, backup qlib_dir to backup_dirfreq: str, default \"day\"transaction frequencymax_workers:\ \ int, default Nonenumber of threadsdate_field_name: str, default \"date\"the\ \ name of the date field in the csvfile_suffix: str, default \".csv\"file suffixsymbol_field_name:\ \ str, default \"symbol\"symbol field nameinclude_fields: tupledump fieldsexclude_fields:\ \ tuplefields not dumpedlimit_nums: intUse when debugging, default None" function: 'dump_bin.DumpDataBase:' - docstring: null function: dump_bin._backup_qlib_dir - docstring: null function: dump_bin._format_datetime - docstring: null function: dump_bin._get_date - docstring: null function: dump_bin._get_source_data - docstring: null function: dump_bin.get_symbol_from_file - docstring: null function: dump_bin.get_dump_fields - docstring: null function: dump_bin._read_calendars - docstring: null function: dump_bin._read_instruments - docstring: null function: dump_bin.save_calendars - docstring: null function: dump_bin.save_instruments - docstring: null function: dump_bin.data_merge_calendar - docstring: null function: dump_bin.get_datetime_index - docstring: null function: dump_bin._data_to_bin - docstring: null function: dump_bin._dump_bin - docstring: null function: dump_bin.dump - docstring: null function: dump_bin.DumpDataAll - docstring: null function: dump_bin._get_all_date - docstring: null function: dump_bin._dump_calendars - docstring: null function: dump_bin._dump_instruments - docstring: null function: dump_bin._dump_features - docstring: null function: dump_bin.dump - docstring: null function: dump_bin.DumpDataFix - docstring: null function: dump_bin._dump_instruments - docstring: null function: dump_bin.dump - docstring: " \nParameters----------csv_path: strstock data path or directoryqlib_dir:\ \ strqlib(dump) data directorbackup_dir: str, default Noneif backup_dir is not\ \ None, backup qlib_dir to backup_dirfreq: str, default \"day\"transaction frequencymax_workers:\ \ int, default Nonenumber of threadsdate_field_name: str, default \"date\"the\ \ name of the date field in the csvfile_suffix: str, default \".csv\"file suffixsymbol_field_name:\ \ str, default \"symbol\"symbol field nameinclude_fields: tupledump fieldsexclude_fields:\ \ tuplefields not dumpedlimit_nums: intUse when debugging, default None" function: dump_bin.DumpDataUpdate - docstring: null function: dump_bin._load_all_source_data - docstring: null function: dump_bin._read_csv - docstring: null function: dump_bin._dump_calendars - docstring: null function: dump_bin._dump_instruments - docstring: null function: dump_bin._dump_features - docstring: " \nParameters----------qlib_dir : strqlib dircsv_path : strorigin\ \ csv pathcheck_fields : str, optionalcheck fields, by default None, check qlib_dir/features//*..binfreq\ \ : str, optionalfreq, value from [\"day\", \"1m\"]symbol_field_name: str, optionalsymbol\ \ field name, by default \"symbol\"date_field_name: str, optionaldate field name,\ \ by default \"date\"file_suffix: str, optionalcsv file suffix, by default \"\ .csv\"max_workers: int, optionalmax workers, by default 16" function: 'check_dump_bin.CheckBin:' - docstring: null function: check_dump_bin._compare - docstring: " \nParameters----------csv_path: strstock data path or directoryqlib_dir:\ \ strqlib(dump) data directorbackup_dir: str, default Noneif backup_dir is not\ \ None, backup qlib_dir to backup_dirfreq: str, default \"quarterly\"data frequencymax_workers:\ \ int, default Nonenumber of threadsdate_column_name: str, default \"date\"the\ \ name of the date field in the csvfile_suffix: str, default \".csv\"file suffixinclude_fields:\ \ tupledump fieldsexclude_fields: tuplefields not dumpedlimit_nums: intUse when\ \ debugging, default None" function: 'dump_pit.DumpPitData:' - docstring: null function: dump_pit._backup_qlib_dir - docstring: null function: dump_pit.get_source_data - docstring: null function: dump_pit.get_symbol_from_file - docstring: null function: dump_pit.get_dump_fields - docstring: null function: dump_pit.get_filenames - docstring: " \ndump data as the following format:`/path/to/.data`[date,\ \ period, value, _next][date, period, value, _next][...]`/path/to/.index`[first_year,\ \ index, index, ...]`` contains the data as the point-in-time (PIT)\ \ order: `value` of `period`is published at `date`, and its successive revised\ \ value can be found at `_next` (linked list).`.index` contains the index\ \ of value for each period (quarter or year). To savedisk space, we only store\ \ the `first_year` as its followings periods can be easily infered.Parameters----------symbol:\ \ strstock symbolinterval: strdata intervaloverwrite: boolwhether overwrite existing\ \ data or update only" function: dump_pit._dump_pit - docstring: ' get SH/SZ history calendar list Parameters----------bench_code: strvalue from ["CSI300", "CSI500", "ALL", "US_ALL"]Returns-------history calendar list' function: utils.get_calendar_list - docstring: null function: utils._get_calendar - docstring: null function: utils._get_calendar - docstring: null function: utils.return_date_list - docstring: ' get calendar list by selecting the date when few funds trade in this day Parameters----------source_dir: str or PathThe directory where the raw data collected from the Internet is saveddate_field_name: strdate field name, default is datethreshold: floatthreshold to exclude some days when few funds trade in this day, default 0.5minimum_count: intminimum count of funds should trade in one daymax_workers: intConcurrent number, default is 16Returns-------history calendar list' function: utils.get_calendar_list_by_ratio - docstring: ' get SH/SZ stock symbols Returns-------stock symbols' function: utils.get_hs_stock_symbols - docstring: null function: utils._get_symbol - docstring: ' get US stock symbols Returns-------stock symbols' function: utils.get_us_stock_symbols - docstring: null function: utils._get_eastmoney - docstring: null function: utils._get_nasdaq - docstring: null function: utils._get_nyse - docstring: null function: utils._format - docstring: ' get IN stock symbols Returns-------stock symbols' function: utils.get_in_stock_symbols - docstring: null function: utils._get_nifty - docstring: null function: utils._format - docstring: ' get Brazil(B3) stock symbols Returns-------B3 stock symbols' function: utils.get_br_stock_symbols - docstring: null function: utils._get_ibovespa - docstring: null function: utils._format - docstring: ' get en fund symbols Returns-------fund symbols in China' function: utils.get_en_fund_symbols - docstring: null function: utils._get_eastmoney - docstring: ' symbol suffix to prefix Parameters----------symbol: strsymbolcapital : boolby default TrueReturns-------' function: utils.symbol_suffix_to_prefix - docstring: ' symbol prefix to sufix Parameters----------symbol: strsymbolcapital : boolby default TrueReturns-------' function: utils.symbol_prefix_to_sufix - docstring: null function: utils.deco_retry - docstring: null function: utils.deco_func - docstring: null function: utils.wrapper - docstring: ' get trading date by shift Parameters----------trading_list: listtrading calendar listshift : intshift, default is 1trading_date : pd.Timestamptrading dateReturns-------' function: utils.get_trading_date_by_shift - docstring: ' generate minutes calendar Parameters----------calendars: Iterabledaily calendarfreq: strby default 1minam_range: Tuple[str, str]AM Time Range, by default China-Stock: ("09:30:00", "11:29:00")pm_range: Tuple[str, str]PM Time Range, by default China-Stock: ("13:00:00", "14:59:00")' function: utils.generate_minutes_calendar_from_daily - docstring: " \nParameters----------qlib_dir: strqlib data dir, default \"Path(__file__).parent/qlib_data\"\ index_name: strindex name, value from [\"csi100\", \"csi300\"]method: strmethod,\ \ value from [\"parse_instruments\", \"save_new_companies\"]freq: strfreq, value\ \ from [\"day\", \"1min\"]request_retry: intrequest retry, by default 5retry_sleep:\ \ intrequest sleep, by default 3market_index: strWhere the files to obtain the\ \ index are located,for example data_collector.cn_index.collectorExamples-------#\ \ parse instruments$ python collector.py --index_name CSI300 --qlib_dir ~/.qlib/qlib_data/cn_data\ \ --method parse_instruments# parse new companies$ python collector.py --index_name\ \ CSI300 --qlib_dir ~/.qlib/qlib_data/cn_data --method save_new_companies" function: utils.get_instruments - docstring: " \nParameters----------save_dir: strinstrument save dirmax_workers:\ \ intworkers, default 1; Concurrent number, default is 1; when collecting data,\ \ it is recommended that max_workers be set to 1max_collector_count: intdefault\ \ 2delay: floattime.sleep(delay), default 0interval: strfreq, value from [1min,\ \ 1d], default 1dstart: strstart datetime, default Noneend: strend datetime, default\ \ Nonecheck_data_length: intcheck data length, if not None and greater than 0,\ \ each symbol will be considered complete if its data length is greater than or\ \ equal to this value, otherwise it will be fetched again, the maximum number\ \ of fetches being (max_collector_count). By default None.limit_nums: intusing\ \ for debug, by default None" function: base.BaseCollector - docstring: null function: base.normalize_start_datetime - docstring: null function: base.normalize_end_datetime - docstring: null function: base.get_instrument_list - docstring: ' normalize symbol raise NotImplementedError("rewrite normalize_symbol")@abc.abstractmethod' function: base.normalize_symbol - docstring: ' get data with symbol Parameters----------symbol: strinterval: strvalue from [1min, 1d]start_datetime: pd.Timestampend_datetime: pd.TimestampReturns---------pd.DataFrame, "symbol" and "date"in pd.columns' function: base.get_data - docstring: null function: base.sleep - docstring: " \nParameters----------symbol: str" function: base._simple_collector - docstring: ' save instrument data to file Parameters----------symbol: strinstrument codedf : pd.DataFramedf.columns must contain "symbol" and "datetime"' function: base.save_instrument - docstring: null function: base.cache_small_data - docstring: null function: base._collector - docstring: ' collector data logger.info("start collector data......")instrument_list = self.instrument_listfor i in range(self.max_collector_count):if not instrument_list:breaklogger.info(f"getting data: {i+1}")instrument_list = self._collector(instrument_list)logger.info(f"{i+1} finish.")for _symbol, _df_list in self.mini_symbol_map.items():_df = pd.concat(_df_list, sort=False)if not _df.empty:self.save_instrument(_symbol, _df.drop_duplicates(["date"]).sort_values(["date"]))if self.mini_symbol_map:logger.warning(f"less than {self.check_data_length} instrument list: {list(self.mini_symbol_map.keys())}")logger.info(f"total {len(self.instrument_list)}, error: {len(set(instrument_list))}")' function: base.collector_data - docstring: " \nParameters----------date_field_name: strdate field name, default\ \ is datesymbol_field_name: strsymbol field name, default is symbol" function: base.BaseNormalize - docstring: null function: base.normalize - docstring: ' Get benchmark calendar raise NotImplementedError("")' function: base._get_calendar_list - docstring: " \nParameters----------source_dir: str or PathThe directory where\ \ the raw data collected from the Internet is savedtarget_dir: str or PathDirectory\ \ for normalize datanormalize_class: Type[YahooNormalize]normalize classmax_workers:\ \ intConcurrent number, default is 16date_field_name: strdate field name, default\ \ is datesymbol_field_name: strsymbol field name, default is symbol" function: 'base.Normalize:' - docstring: null function: base._executor - docstring: null function: base.normalize - docstring: " \nParameters----------source_dir: strThe directory where the\ \ raw data collected from the Internet is saved, default \"Path(__file__).parent/source\"\ normalize_dir: strDirectory for normalize data, default \"Path(__file__).parent/normalize\"\ max_workers: intConcurrent number, default is 1; Concurrent number, default is\ \ 1; when collecting data, it is recommended that max_workers be set to 1interval:\ \ strfreq, value from [1min, 1d], default 1d" function: base.BaseRun - docstring: null function: base.collector_class_name - docstring: null function: base.normalize_class_name - docstring: null function: base.default_base_dir - docstring: ' download data from Internet Parameters----------max_collector_count: intdefault 2delay: floattime.sleep(delay), default 0start: strstart datetime, default "2000-01-01"end: strend datetime, default ``pd.Timestamp(datetime.datetime.now() + pd.Timedelta(days=1))``check_data_length: intcheck data length, if not None and greater than 0, each symbol will be considered complete if its data length is greater than or equal to this value, otherwise it will be fetched again, the maximum number of fetches being (max_collector_count). By default None.limit_nums: intusing for debug, by default NoneExamples---------# get daily data$ python collector.py download_data --source_dir ~/.qlib/instrument_data/source --region CN --start 2020-11-01 --end 2020-11-10 --delay 0.1 --interval 1d# get 1m data$ python collector.py download_data --source_dir ~/.qlib/instrument_data/source --region CN --start 2020-11-01 --end 2020-11-10 --delay 0.1 --interval 1m' function: base.download_data - docstring: ' normalize data Parameters----------date_field_name: strdate field name, default datesymbol_field_name: strsymbol field name, default symbolExamples---------$ python collector.py normalize_data --source_dir ~/.qlib/instrument_data/source --normalize_dir ~/.qlib/instrument_data/normalize --region CN --interval 1d' function: base.normalize_data - docstring: " \nParameters----------index_name: strindex nameqlib_dir: strqlib\ \ directory, by default Path(__file__).resolve().parent.joinpath(\"qlib_data\"\ )freq: strfreq, value from [\"day\", \"1min\"]request_retry: intrequest retry,\ \ by default 5retry_sleep: intrequest sleep, by default 3" function: 'index.IndexBase:' - docstring: " \nReturns-------index start date" function: index.bench_start_date - docstring: ' get history trading date Returns-------calendar list' function: index.calendar_list - docstring: " \nReturns-------pd.DataFrame:symbol start_date end_dateSH600000\ \ 2000-01-01 2099-12-31dtypes:symbol: strstart_date: pd.Timestampend_date:\ \ pd.Timestamp" function: index.get_new_companies - docstring: ' get companies changes Returns-------pd.DataFrame:symbol date typeSH600000 2019-11-11 addSH600000 2020-11-10 removedtypes:symbol: strdate: pd.Timestamptype: str, value from ["add", "remove"]' function: index.get_changes - docstring: ' formatting the datetime in an instrument Parameters----------inst_df: pd.DataFrameinst_df.columns = [self.SYMBOL_FIELD_NAME, self.START_DATE_FIELD, self.END_DATE_FIELD]Returns-------' function: index.format_datetime - docstring: ' save new companies Examples-------$ python collector.py save_new_companies --index_name CSI300 --qlib_dir ~/.qlib/qlib_data/cn_data' function: index.save_new_companies - docstring: ' get changes with history companies Parameters----------history_companies : pd.DataFramesymbol dateSH600000 2020-11-11dtypes:symbol: strdate: pd.TimestampReturn--------pd.DataFrame:symbol date typeSH600000 2019-11-11 addSH600000 2020-11-10 removedtypes:symbol: strdate: pd.Timestamptype: str, value from ["add", "remove"]' function: index.get_changes_with_history_companies - docstring: ' parse instruments, eg: csi300.txt Examples-------$ python collector.py parse_instruments --index_name CSI300 --qlib_dir ~/.qlib/qlib_data/cn_data' function: index.parse_instruments - docstring: " \nParameters----------qlib_dir:qlib data directorystart_datestart\ \ dateend_dateend date" function: 'future_calendar_collector.CollectorFutureCalendar:' - docstring: null function: future_calendar_collector.calendar_list - docstring: null function: future_calendar_collector._format_datetime - docstring: null function: future_calendar_collector.write_calendar - docstring: " \nReturns-------" function: future_calendar_collector.collector - docstring: null function: future_calendar_collector.CollectorFutureCalendarCN - docstring: null function: future_calendar_collector.collector - docstring: null function: future_calendar_collector.CollectorFutureCalendarUS - docstring: null function: future_calendar_collector.collector - docstring: ' Collect future calendar(day) Parameters----------qlib_dir:qlib data directoryregion:cn/CN or us/USstart_datestart dateend_dateend dateExamples-------# get cn future calendar$ python future_calendar_collector.py --qlib_data_1d_dir --region cn' function: future_calendar_collector.run - docstring: ' get crypto symbols in coingecko Returns-------crypto symbols in given exchanges list of coingecko' function: collector.get_cg_crypto_symbols - docstring: null function: collector._get_coingecko - docstring: " \nParameters----------save_dir: strcrypto save dirmax_workers:\ \ intworkers, default 4max_collector_count: intdefault 2delay: floattime.sleep(delay),\ \ default 0interval: strfreq, value from [1min, 1d], default 1minstart: strstart\ \ datetime, default Noneend: strend datetime, default Nonecheck_data_length: intcheck\ \ data length, if not None and greater than 0, each symbol will be considered\ \ complete if its data length is greater than or equal to this value, otherwise\ \ it will be fetched again, the maximum number of fetches being (max_collector_count).\ \ By default None.limit_nums: intusing for debug, by default None" function: collector.CryptoCollector - docstring: null function: collector.init_datetime - docstring: null function: collector.convert_datetime - docstring: null function: collector._timezone - docstring: null function: collector.get_data_from_remote - docstring: null function: collector.get_data - docstring: null function: collector._get_simple - docstring: null function: collector.CryptoCollector1d - docstring: null function: collector.get_instrument_list - docstring: null function: collector.normalize_symbol - docstring: null function: collector._timezone - docstring: null function: collector.CryptoNormalize - docstring: null function: collector.normalize_crypto - docstring: null function: collector.normalize - docstring: null function: collector.CryptoNormalize1d - docstring: null function: collector._get_calendar_list - docstring: " \nParameters----------source_dir: strThe directory where the\ \ raw data collected from the Internet is saved, default \"Path(__file__).parent/source\"\ normalize_dir: strDirectory for normalize data, default \"Path(__file__).parent/normalize\"\ max_workers: intConcurrent number, default is 1interval: strfreq, value from [1min,\ \ 1d], default 1d" function: collector.Run - docstring: null function: collector.collector_class_name - docstring: null function: collector.normalize_class_name - docstring: null function: collector.default_base_dir - docstring: ' download data from Internet Parameters----------max_collector_count: intdefault 2delay: floattime.sleep(delay), default 0interval: strfreq, value from [1min, 1d], default 1d, currently only supprot 1dstart: strstart datetime, default "2000-01-01"end: strend datetime, default ``pd.Timestamp(datetime.datetime.now() + pd.Timedelta(days=1))``check_data_length: int # if this param useful?check data length, if not None and greater than 0, each symbol will be considered complete if its data length is greater than or equal to this value, otherwise it will be fetched again, the maximum number of fetches being (max_collector_count). By default None.limit_nums: intusing for debug, by default NoneExamples---------# get daily data$ python collector.py download_data --source_dir ~/.qlib/crypto_data/source/1d --start 2015-01-01 --end 2021-11-30 --delay 1 --interval 1d' function: collector.download_data - docstring: ' normalize data Parameters----------date_field_name: strdate field name, default datesymbol_field_name: strsymbol field name, default symbolExamples---------$ python collector.py normalize_data --source_dir ~/.qlib/crypto_data/source/1d --normalize_dir ~/.qlib/crypto_data/source/1d_nor --interval 1d --date_field_name date' function: collector.normalize_data - docstring: null function: collector.WIKIIndex - docstring: " \nReturns-------index start date" function: collector.bench_start_date - docstring: ' get companies changes Returns-------pd.DataFrame:symbol date typeSH600000 2019-11-11 addSH600000 2020-11-10 removedtypes:symbol: strdate: pd.Timestamptype: str, value from ["add", "remove"]' function: collector.get_changes - docstring: ' formatting the datetime in an instrument Parameters----------inst_df: pd.DataFrameinst_df.columns = [self.SYMBOL_FIELD_NAME, self.START_DATE_FIELD, self.END_DATE_FIELD]Returns-------' function: collector.format_datetime - docstring: ' get history trading date Returns-------calendar list' function: collector.calendar_list - docstring: null function: collector._request_new_companies - docstring: null function: collector.set_default_date_range - docstring: null function: collector.get_new_companies - docstring: null function: collector.filter_df - docstring: null function: collector.NASDAQ100Index - docstring: null function: collector.filter_df - docstring: null function: collector.bench_start_date - docstring: null function: collector._request_history_companies - docstring: null function: collector.get_history_companies - docstring: null function: collector.get_changes - docstring: null function: collector.DJIAIndex - docstring: null function: collector.bench_start_date - docstring: null function: collector.get_changes - docstring: null function: collector.filter_df - docstring: null function: collector.parse_instruments - docstring: null function: collector.SP500Index - docstring: null function: collector.bench_start_date - docstring: null function: collector.get_changes - docstring: null function: collector.filter_df - docstring: null function: collector.SP400Index - docstring: null function: collector.bench_start_date - docstring: null function: collector.get_changes - docstring: null function: collector.filter_df - docstring: " \nParameters----------save_dir: strinstrument save dirmax_workers:\ \ intworkers, default 1; Concurrent number, default is 1; when collecting data,\ \ it is recommended that max_workers be set to 1max_collector_count: intdefault\ \ 2delay: floattime.sleep(delay), default 0interval: strfreq, value from [1min,\ \ 1d], default 1dstart: strstart datetime, default Noneend: strend datetime, default\ \ Nonecheck_data_length: intcheck data length, if not None and greater than 0,\ \ each symbol will be considered complete if its data length is greater than or\ \ equal to this value, otherwise it will be fetched again, the maximum number\ \ of fetches being (max_collector_count). By default None.limit_nums: intusing\ \ for debug, by default Nonesymbol_regex: strsymbol regular expression, by default\ \ None." function: collector.PitCollector - docstring: null function: collector.get_instrument_list - docstring: null function: collector.normalize_symbol - docstring: null function: collector.get_performance_express_report_df - docstring: null function: collector.get_profit_df - docstring: null function: collector.get_forecast_report_df - docstring: null function: collector.get_growth_df - docstring: null function: collector.get_data - docstring: null function: collector.PitNormalize - docstring: null function: collector.normalize - docstring: null function: collector._get_calendar_list - docstring: null function: collector.Run - docstring: null function: collector.collector_class_name - docstring: null function: collector.normalize_class_name - docstring: null function: fill_cn_1min_data.get_date_range - docstring: null function: fill_cn_1min_data.get_symbols - docstring: ' Use 1d data to fill in the missing symbols relative to 1min Parameters----------data_1min_dir: str1min data dirqlib_data_1d_dir: str1d qlib data(bin data) dir, from: https://qlib.readthedocs.io/en/latest/component/data.html#converting-csv-format-into-qlib-formatmax_workers: intThreadPoolExecutor(max_workers), by default 16date_field_name: strdate field name, by default datesymbol_field_name: strsymbol field name, by default symbol' function: fill_cn_1min_data.fill_1min_using_1d - docstring: null function: future_trading_date_collector.read_calendar_from_qlib - docstring: null function: future_trading_date_collector.write_calendar_to_qlib - docstring: null function: future_trading_date_collector.generate_qlib_calendar - docstring: ' get future calendar Parameters----------qlib_dir: str or Pathqlib data directoryfreq: strvalue from ["day", "1min"], by default day' function: future_trading_date_collector.future_calendar_collector - docstring: null function: collector.retry_request - docstring: null function: collector.CSIIndex - docstring: ' get history trading date Returns-------calendar list' function: collector.calendar_list - docstring: null function: collector.new_companies_url - docstring: null function: collector.changes_url - docstring: " \nReturns-------index start date" function: collector.bench_start_date - docstring: " \nReturns-------index code" function: collector.index_code - docstring: ' Which table of changes in html CSI300: 0CSI100: 1:return:' function: collector.html_table_index - docstring: ' formatting the datetime in an instrument Parameters----------inst_df: pd.DataFrameinst_df.columns = [self.SYMBOL_FIELD_NAME, self.START_DATE_FIELD, self.END_DATE_FIELD]Returns-------' function: collector.format_datetime - docstring: ' get companies changes Returns-------pd.DataFrame:symbol date typeSH600000 2019-11-11 addSH600000 2020-11-10 removedtypes:symbol: strdate: pd.Timestamptype: str, value from ["add", "remove"]' function: collector.get_changes - docstring: " \nParameters----------symbol: strsymbolReturns-------symbol" function: collector.normalize_symbol - docstring: null function: collector._parse_excel - docstring: null function: collector._parse_table - docstring: ' read change from url The parameter url is from the _get_change_notices_url method.Determine the stock add_date/remove_date based on the title.The response contains three cases:1.Only excel_url(extract data from excel_url)2.Both the excel_url and the body text(try to extract data from excel_url first, and then try to extract data from body text)3.Only body text(extract data from body text)Parameters----------url : strchange urlReturns-------pd.DataFrame:symbol date typeSH600000 2019-11-11 addSH600000 2020-11-10 removedtypes:symbol: strdate: pd.Timestamptype: str, value from ["add", "remove"]' function: collector._read_change_from_url - docstring: ' get change notices url Returns-------[url1, url2]' function: collector._get_change_notices_url - docstring: " \nReturns-------pd.DataFrame:symbol start_date end_dateSH600000\ \ 2000-01-01 2099-12-31dtypes:symbol: strstart_date: pd.Timestampend_date:\ \ pd.Timestamp" function: collector.get_new_companies - docstring: null function: collector.CSI300Index - docstring: null function: collector.index_code - docstring: null function: collector.bench_start_date - docstring: null function: collector.html_table_index - docstring: null function: collector.CSI100Index - docstring: null function: collector.index_code - docstring: null function: collector.bench_start_date - docstring: null function: collector.html_table_index - docstring: null function: collector.CSI500Index - docstring: null function: collector.index_code - docstring: null function: collector.bench_start_date - docstring: ' get companies changes Return--------pd.DataFrame:symbol date typeSH600000 2019-11-11 addSH600000 2020-11-10 removedtypes:symbol: strdate: pd.Timestamptype: str, value from ["add", "remove"]' function: collector.get_changes - docstring: " \nReturns-------pd.DataFrame:symbol date typeSH600000\ \ 2019-11-11 addSH600000 2020-11-10 removedtypes:symbol: strdate: pd.Timestamptype:\ \ str, value from [\"add\", \"remove\"]" function: collector.get_history_companies - docstring: " \nData source: http://baostock.com/baostock/index.php/%E4%B8%AD%E8%AF%81500%E6%88%90%E5%88%86%E8%82%A1Avoid\ \ a large number of parallel data acquisition,such as 1000 times of concurrent\ \ data acquisition, because IP will be blockedReturns-------pd.DataFrame:date\ \ symbol code_nameSH600039 2007-01-15 \u56DB\u5DDD\u8DEF\u6865\ SH600051 2020-01-15 \u5B81\u6CE2\u8054\u5408dtypes:date: pd.Timestampsymbol:\ \ strcode_name: str" function: collector.get_data_from_baostock - docstring: " \nReturns-------pd.DataFrame:symbol start_date end_dateSH600000\ \ 2000-01-01 2099-12-31dtypes:symbol: strstart_date: pd.Timestampend_date:\ \ pd.Timestamp" function: collector.get_new_companies - docstring: null function: collector.IBOVIndex - docstring: " \nThe ibovespa index started on 2 January 1968 (wiki), however,no\ \ suitable data source that keeps track of ibovespa's historystocks composition\ \ has been found. Except from the repo indicatedin README. Which keeps track of\ \ such information starting fromthe first quarter of 2003" function: collector.bench_start_date - docstring: " \nThis function is used to calculated what is the currentfour\ \ month period for the current month. For example,If the current month is August\ \ 8, its four month periodis 2Q.OBS: In english Q is used to represent *quarter*which\ \ means a three month period. However, inportuguese we use Q to represent a four\ \ month period.In other words,Jan, Feb, Mar, Apr: 1QMay, Jun, Jul, Aug: 2QSep,\ \ Oct, Nov, Dez: 3QParameters----------month : intCurrent month (1 <= month <=\ \ 12)Returns-------current_4m_period:strCurrent Four Month Period (1Q or 2Q or\ \ 3Q)" function: collector.get_current_4_month_period - docstring: " \nThe ibovespa index is updated every four months.Therefore,\ \ we will represent each time period as 2003_1Qwhich means 2003 first four mount\ \ period (Jan, Feb, Mar, Apr)" function: collector.get_four_month_period - docstring: ' formatting the datetime in an instrument Parameters----------inst_df: pd.DataFrameinst_df.columns = [self.SYMBOL_FIELD_NAME, self.START_DATE_FIELD, self.END_DATE_FIELD]Returns-------inst_df: pd.DataFrame' function: collector.format_datetime - docstring: " \nParameters----------cell: strIt must be on the format 2003_1Q\ \ --> years_4_month_periodsReturns----------date: strReturns date in format 2003-03-01" function: collector.format_quarter - docstring: " \nAccess the index historic composition and compare it quarterby\ \ quarter and year by year in order to generate a file thatkeeps track of which\ \ stocks have been removed and which havebeen added.The Dataframe used as reference\ \ will provided the indexcomposition for each year an quarter:pd.DataFrame:symbolSH600000SH600001...Parameters----------self:\ \ is used to represent the instance of the class.Returns----------pd.DataFrame:symbol\ \ date typeSH600000 2019-11-11 addSH600001 2020-11-10 removedtypes:symbol:\ \ strdate: pd.Timestamptype: str, value from [\"add\", \"remove\"]" function: collector.get_changes - docstring: " \nGet latest index composition.The repo indicated on README\ \ has implemented a scriptto get the latest index composition from B3 website\ \ usingselenium. Therefore, this method will download the filecontaining such\ \ compositionParameters----------self: is used to represent the instance of the\ \ class.Returns----------pd.DataFrame:symbol start_date end_dateRRRP3\t\ \ 2020-11-13\t2022-03-02ALPA4\t 2008-01-02\t2022-03-02dtypes:symbol: strstart_date:\ \ pd.Timestampend_date: pd.Timestamp" function: collector.get_new_companies - docstring: " \nParameters----------save_dir: strstock save dirmax_workers:\ \ intworkers, default 4max_collector_count: intdefault 2delay: floattime.sleep(delay),\ \ default 0interval: strfreq, value from [1min, 1d], default 1minstart: strstart\ \ datetime, default Noneend: strend datetime, default Nonecheck_data_length: intcheck\ \ data length, by default Nonelimit_nums: intusing for debug, by default None" function: collector.YahooCollector - docstring: null function: collector.init_datetime - docstring: null function: collector.convert_datetime - docstring: null function: collector._timezone - docstring: null function: collector.get_data_from_remote - docstring: null function: collector._show_logging_func - docstring: null function: collector.get_data - docstring: null function: collector._get_simple - docstring: ' collector data super(YahooCollector, self).collector_data()self.download_index_data()@abc.abstractmethod' function: collector.collector_data - docstring: ' download index data raise NotImplementedError("rewrite download_index_data")' function: collector.download_index_data - docstring: null function: collector.YahooCollectorCN - docstring: null function: collector.get_instrument_list - docstring: null function: collector.normalize_symbol - docstring: null function: collector._timezone - docstring: null function: collector.YahooCollectorCN1d - docstring: null function: collector.download_index_data - docstring: null function: collector.YahooCollectorCN1min - docstring: null function: collector.get_instrument_list - docstring: null function: collector.download_index_data - docstring: null function: collector.YahooCollectorUS - docstring: null function: collector.get_instrument_list - docstring: null function: collector.download_index_data - docstring: null function: collector.normalize_symbol - docstring: null function: collector._timezone - docstring: null function: collector.YahooCollectorUS1d - docstring: null function: collector.YahooCollectorUS1min - docstring: null function: collector.YahooCollectorIN - docstring: null function: collector.get_instrument_list - docstring: null function: collector.download_index_data - docstring: null function: collector.normalize_symbol - docstring: null function: collector._timezone - docstring: null function: collector.YahooCollectorIN1d - docstring: null function: collector.YahooCollectorIN1min - docstring: null function: collector.YahooCollectorBR - docstring: " \nThe reason to use retry=2 is due to the fact thatYahoo Finance\ \ unfortunately does not keep track of someBrazilian stocks.Therefore, the decorator\ \ deco_retry with retry argumentset to 5 will keep trying to get the stock data\ \ up to 5 times,which makes the code to download Brazilians stocks very slow.In\ \ future, this may change, but for nowI suggest to leave retry argument to 1 or\ \ 2 inorder to improve download speed.To achieve this goal an abstract attribute\ \ (retry)was added into YahooCollectorBR base class" function: collector.retry - docstring: null function: collector.get_instrument_list - docstring: null function: collector.download_index_data - docstring: null function: collector.normalize_symbol - docstring: null function: collector._timezone - docstring: null function: collector.YahooCollectorBR1d - docstring: null function: collector.YahooCollectorBR1min - docstring: null function: collector.YahooNormalize - docstring: null function: collector.calc_change - docstring: null function: collector.normalize_yahoo - docstring: null function: collector.normalize - docstring: ' adjusted price raise NotImplementedError("rewrite adjusted_price")' function: collector.adjusted_price - docstring: null function: collector.YahooNormalize1d - docstring: null function: collector.adjusted_price - docstring: null function: collector.normalize - docstring: ' get first close value Notes-----For incremental updates(append) to Yahoo 1D data, user need to use a close that is not 0 on the first trading day of the existing data' function: collector._get_first_close - docstring: ' manual adjust data: All fields (except change) are standardized according to the close of the first day if df.empty:return dfdf = df.copy()df.sort_values(self._date_field_name, inplace=True)df = df.set_index(self._date_field_name)_close = self._get_first_close(df)for _col in df.columns:# NOTE: retain original adjclose, required for incremental updatesif _col in [self._symbol_field_name, "adjclose", "change"]:continueif _col == "volume":df[_col] = df[_col] * _closeelse:df[_col] = df[_col] / _closereturn df.reset_index()' function: collector._manual_adj_data - docstring: " \nParameters----------old_qlib_data_dir: str, Paththe qlib data\ \ to be updated for yahoo, usually from: https://github.com/microsoft/qlib/tree/main/scripts#download-cn-datadate_field_name:\ \ strdate field name, default is datesymbol_field_name: strsymbol field name,\ \ default is symbol" function: collector.YahooNormalize1dExtend - docstring: null function: collector._get_old_data - docstring: null function: collector._get_close - docstring: null function: collector._get_first_close - docstring: null function: collector._get_last_close - docstring: null function: collector._get_last_date - docstring: null function: collector.normalize - docstring: null function: collector.YahooNormalize1min - docstring: null function: collector.calendar_list_1d - docstring: null function: collector.generate_1min_from_daily - docstring: ' get 1d data Returns------data_1d: pd.DataFramedata_1d.columns = [self._date_field_name, self._symbol_field_name, "paused", "volume", "factor", "close"]' function: collector.get_1d_data - docstring: null function: collector.adjusted_price - docstring: null function: collector._calc_factor - docstring: null function: collector.calc_paused_num - docstring: null function: collector.symbol_to_yahoo - docstring: null function: collector._get_1d_calendar_list - docstring: ' Normalised to 1min using local 1d data self, qlib_data_1d_dir: [str, Path], date_field_name: str = "date", symbol_field_name: str = "symbol", **kwargs):' function: collector.YahooNormalize1minOffline - docstring: null function: collector._get_1d_calendar_list - docstring: null function: collector._get_all_1d_data - docstring: ' get 1d data Returns------data_1d: pd.DataFramedata_1d.columns = [self._date_field_name, self._symbol_field_name, "paused", "volume", "factor", "close"]' function: collector.get_1d_data - docstring: null function: 'collector.YahooNormalizeUS:' - docstring: null function: collector._get_calendar_list - docstring: null function: collector.YahooNormalizeUS1d - docstring: null function: collector.YahooNormalizeUS1dExtend - docstring: null function: collector.YahooNormalizeUS1min - docstring: null function: collector._get_calendar_list - docstring: null function: collector._get_1d_calendar_list - docstring: null function: collector.symbol_to_yahoo - docstring: null function: 'collector.YahooNormalizeIN:' - docstring: null function: collector._get_calendar_list - docstring: null function: collector.YahooNormalizeIN1d - docstring: null function: collector.YahooNormalizeIN1min - docstring: null function: collector._get_calendar_list - docstring: null function: collector._get_1d_calendar_list - docstring: null function: collector.symbol_to_yahoo - docstring: null function: 'collector.YahooNormalizeCN:' - docstring: null function: collector._get_calendar_list - docstring: null function: collector.YahooNormalizeCN1d - docstring: null function: collector.YahooNormalizeCN1dExtend - docstring: null function: collector.YahooNormalizeCN1min - docstring: null function: collector._get_calendar_list - docstring: null function: collector.symbol_to_yahoo - docstring: null function: collector._get_1d_calendar_list - docstring: null function: 'collector.YahooNormalizeBR:' - docstring: null function: collector._get_calendar_list - docstring: null function: collector.YahooNormalizeBR1d - docstring: null function: collector.YahooNormalizeBR1min - docstring: null function: collector._get_calendar_list - docstring: null function: collector._get_1d_calendar_list - docstring: null function: collector.symbol_to_yahoo - docstring: " \nParameters----------source_dir: strThe directory where the\ \ raw data collected from the Internet is saved, default \"Path(__file__).parent/source\"\ normalize_dir: strDirectory for normalize data, default \"Path(__file__).parent/normalize\"\ max_workers: intConcurrent number, default is 1; when collecting data, it is recommended\ \ that max_workers be set to 1interval: strfreq, value from [1min, 1d], default\ \ 1dregion: strregion, value from [\"CN\", \"US\", \"BR\"], default \"CN\"" function: collector.Run - docstring: null function: collector.collector_class_name - docstring: null function: collector.normalize_class_name - docstring: null function: collector.default_base_dir - docstring: ' download data from Internet Parameters----------max_collector_count: intdefault 2delay: floattime.sleep(delay), default 0.5start: strstart datetime, default "2000-01-01"; closed interval(including start)end: strend datetime, default ``pd.Timestamp(datetime.datetime.now() + pd.Timedelta(days=1))``; open interval(excluding end)check_data_length: intcheck data length, if not None and greater than 0, each symbol will be considered complete if its data length is greater than or equal to this value, otherwise it will be fetched again, the maximum number of fetches being (max_collector_count). By default None.limit_nums: intusing for debug, by default NoneNotes-----check_data_length, example:daily, one year: 252 // 4us 1min, a week: 6.5 * 60 * 5cn 1min, a week: 4 * 60 * 5Examples---------# get daily data$ python collector.py download_data --source_dir ~/.qlib/stock_data/source --region CN --start 2020-11-01 --end 2020-11-10 --delay 0.1 --interval 1d# get 1m data$ python collector.py download_data --source_dir ~/.qlib/stock_data/source --region CN --start 2020-11-01 --end 2020-11-10 --delay 0.1 --interval 1m' function: collector.download_data - docstring: ' normalize data Parameters----------date_field_name: strdate field name, default datesymbol_field_name: strsymbol field name, default symbolend_date: strif not None, normalize the last date saved (including end_date); if None, it will ignore this parameter; by default Noneqlib_data_1d_dir: strif interval==1min, qlib_data_1d_dir cannot be None, normalize 1min needs to use 1d data;qlib_data_1d can be obtained like this:$ python scripts/get_data.py qlib_data --target_dir --interval 1d$ python scripts/data_collector/yahoo/collector.py update_data_to_bin --qlib_data_1d_dir --trading_date 2021-06-01or:download 1d data, reference: https://github.com/microsoft/qlib/tree/main/scripts/data_collector/yahoo#1d-from-yahooExamples---------$ python collector.py normalize_data --source_dir ~/.qlib/stock_data/source --normalize_dir ~/.qlib/stock_data/normalize --region cn --interval 1d$ python collector.py normalize_data --qlib_data_1d_dir ~/.qlib/qlib_data/cn_data --source_dir ~/.qlib/stock_data/source_cn_1min --normalize_dir ~/.qlib/stock_data/normalize_cn_1min --region CN --interval 1min' function: collector.normalize_data - docstring: ' normalize data extend; extending yahoo qlib data(from: https://github.com/microsoft/qlib/tree/main/scripts#download-cn-data) Notes-----Steps to extend yahoo qlib data:1. download qlib data: https://github.com/microsoft/qlib/tree/main/scripts#download-cn-data; save to 2. collector source data: https://github.com/microsoft/qlib/tree/main/scripts/data_collector/yahoo#collector-data; save to 3. normalize new source data(from step 2): python scripts/data_collector/yahoo/collector.py normalize_data_1d_extend --old_qlib_dir --source_dir --normalize_dir --region CN --interval 1d4. dump data: python scripts/dump_bin.py dump_update --csv_path --qlib_dir --freq day --date_field_name date --symbol_field_name symbol --exclude_fields symbol,date5. update instrument(eg. csi300): python python scripts/data_collector/cn_index/collector.py --index_name CSI300 --qlib_dir --method parse_instrumentsParameters----------old_qlib_data_dir: strthe qlib data to be updated for yahoo, usually from: https://github.com/microsoft/qlib/tree/main/scripts#download-cn-datadate_field_name: strdate field name, default datesymbol_field_name: strsymbol field name, default symbolExamples---------$ python collector.py normalize_data_1d_extend --old_qlib_dir ~/.qlib/qlib_data/cn_data --source_dir ~/.qlib/stock_data/source --normalize_dir ~/.qlib/stock_data/normalize --region CN --interval 1d' function: collector.normalize_data_1d_extend - docstring: ' download today data from Internet Parameters----------max_collector_count: intdefault 2delay: floattime.sleep(delay), default 0.5check_data_length: intcheck data length, if not None and greater than 0, each symbol will be considered complete if its data length is greater than or equal to this value, otherwise it will be fetched again, the maximum number of fetches being (max_collector_count). By default None.limit_nums: intusing for debug, by default NoneNotes-----Download today''s data:start_time = datetime.datetime.now().date(); closed interval(including start)end_time = pd.Timestamp(start_time + pd.Timedelta(days=1)).date(); open interval(excluding end)check_data_length, example:daily, one year: 252 // 4us 1min, a week: 6.5 * 60 * 5cn 1min, a week: 4 * 60 * 5Examples---------# get daily data$ python collector.py download_today_data --source_dir ~/.qlib/stock_data/source --region CN --delay 0.1 --interval 1d# get 1m data$ python collector.py download_today_data --source_dir ~/.qlib/stock_data/source --region CN --delay 0.1 --interval 1m' function: collector.download_today_data - docstring: ' update yahoo data to bin Parameters----------qlib_data_1d_dir: strthe qlib data to be updated for yahoo, usually from: https://github.com/microsoft/qlib/tree/main/scripts#download-cn-datatrading_date: strtrading days to be updated, by default ``datetime.datetime.now().strftime("%Y-%m-%d")``end_date: strend datetime, default ``pd.Timestamp(trading_date + pd.Timedelta(days=1))``; open interval(excluding end)check_data_length: intcheck data length, if not None and greater than 0, each symbol will be considered complete if its data length is greater than or equal to this value, otherwise it will be fetched again, the maximum number of fetches being (max_collector_count). By default None.delay: floattime.sleep(delay), default 1Notes-----If the data in qlib_data_dir is incomplete, np.nan will be populated to trading_date for the previous trading dayExamples-------$ python collector.py update_data_to_bin --qlib_data_1d_dir --trading_date --end_date # get 1m data' function: collector.update_data_to_bin - docstring: " \nParameters----------save_dir: strfund save dirmax_workers:\ \ intworkers, default 4max_collector_count: intdefault 2delay: floattime.sleep(delay),\ \ default 0interval: strfreq, value from [1min, 1d], default 1minstart: strstart\ \ datetime, default Noneend: strend datetime, default Nonecheck_data_length: intcheck\ \ data length, if not None and greater than 0, each symbol will be considered\ \ complete if its data length is greater than or equal to this value, otherwise\ \ it will be fetched again, the maximum number of fetches being (max_collector_count).\ \ By default None.limit_nums: intusing for debug, by default None" function: collector.FundCollector - docstring: null function: collector.init_datetime - docstring: null function: collector.convert_datetime - docstring: null function: collector._timezone - docstring: null function: collector.get_data_from_remote - docstring: null function: collector.get_data - docstring: null function: collector._get_simple - docstring: null function: collector.FundollectorCN - docstring: null function: collector.get_instrument_list - docstring: null function: collector.normalize_symbol - docstring: null function: collector._timezone - docstring: null function: collector.FundCollectorCN1d - docstring: null function: collector.FundNormalize - docstring: null function: collector.normalize_fund - docstring: null function: collector.normalize - docstring: null function: collector.FundNormalize1d - docstring: null function: 'collector.FundNormalizeCN:' - docstring: null function: collector._get_calendar_list - docstring: null function: collector.FundNormalizeCN1d - docstring: " \nParameters----------source_dir: strThe directory where the\ \ raw data collected from the Internet is saved, default \"Path(__file__).parent/source\"\ normalize_dir: strDirectory for normalize data, default \"Path(__file__).parent/normalize\"\ max_workers: intConcurrent number, default is 4interval: strfreq, value from [1min,\ \ 1d], default 1dregion: strregion, value from [\"CN\"], default \"CN\"" function: collector.Run - docstring: null function: collector.collector_class_name - docstring: null function: collector.normalize_class_name - docstring: null function: collector.default_base_dir - docstring: ' download data from Internet Parameters----------max_collector_count: intdefault 2delay: floattime.sleep(delay), default 0interval: strfreq, value from [1min, 1d], default 1dstart: strstart datetime, default "2000-01-01"end: strend datetime, default ``pd.Timestamp(datetime.datetime.now() + pd.Timedelta(days=1))``check_data_length: int # if this param useful?check data length, if not None and greater than 0, each symbol will be considered complete if its data length is greater than or equal to this value, otherwise it will be fetched again, the maximum number of fetches being (max_collector_count). By default None.limit_nums: intusing for debug, by default NoneExamples---------# get daily data$ python collector.py download_data --source_dir ~/.qlib/fund_data/source/cn_data --region CN --start 2020-11-01 --end 2020-11-10 --delay 0.1 --interval 1d' function: collector.download_data - docstring: ' normalize data Parameters----------date_field_name: strdate field name, default datesymbol_field_name: strsymbol field name, default symbolExamples---------$ python collector.py normalize_data --source_dir ~/.qlib/fund_data/source/cn_data --normalize_dir ~/.qlib/fund_data/source/cn_1d_nor --region CN --interval 1d --date_field_name FSRQ' function: collector.normalize_data - docstring: null function: 'config.Config:' - docstring: null function: config.get - docstring: null function: config.reset - docstring: null function: config.update - docstring: null function: config.set_conf_from_C - docstring: null function: config.register_from_C - docstring: null function: config.QlibConfig - docstring: " \nMotivation:- get the right path (e.g. data uri) for accessing\ \ data based on given information(e.g. provider_uri, mount_path and frequency)-\ \ some helper functions to process uri." function: 'config.DataPathManager:' - docstring: null function: config.format_provider_uri - docstring: null function: config.get_uri_type - docstring: " \nplease refer DataPathManager's __init__ and class doc" function: config.get_data_uri - docstring: null function: config.set_mode - docstring: null function: config.set_region - docstring: null function: config.is_depend_redis - docstring: null function: config.dpm - docstring: null function: config.resolve_path - docstring: " \nconfigure qlib based on the input parametersThe configuration\ \ will act like a dictionary.Normally, it literally is replaced the value according\ \ to the keys.However, sometimes it is hard for users to set the config when the\ \ configuration is nested and complicatedSo this API provides some special parameters\ \ for users to set the keys in a more convenient way.- region: REG_CN, REG_US-\ \ several region-related config will be changedParameters----------default_conf\ \ : strthe default config template chosen by user: \"server\", \"client\"" function: config.set - docstring: null function: config.register - docstring: null function: config.reset_qlib_version - docstring: ' get number of processors given frequency if isinstance(self["kernels"], Callable):return self["kernels"](freq)return self["kernels"]@property' function: config.get_kernels - docstring: " \nInstDictConf is a Dict-based config to describe an instancecase\ \ 1){'class': 'ClassName','kwargs': dict, # It is optional. {} will be used if\ \ not given'model_path': path, # It is optional if module is given in the class}case\ \ 2){'class': ,'kwargs': dict, # It is optional. {} will be\ \ used if not given}" function: typehint.InstDictConf - docstring: " \nParameters----------default_conf: strthe default value is client.\ \ Accepted values: client/server.**kwargs :clear_mem_cache: strthe default value\ \ is True;Will the memory cache be clear.It is often used to improve performance\ \ when init will be called for multiple timesskip_if_reg: bool: strthe default\ \ value is True;When using the recorder, skip_if_reg can set to True to avoid\ \ loss of recorder." function: __init__.init - docstring: null function: __init__._mount_nfs_uri - docstring: ' init_from_yaml_conf :param conf_path: A path to the qlib config in yml format' function: __init__.init_from_yaml_conf - docstring: " \nIf users are building a project follow the following pattern.-\ \ Qlib is a sub folder in project path- There is a file named `config.yaml` in\ \ qlib.For example:If your project file system structure follows such a pattern/-\ \ config.yaml- ...some folders...- qlib/This folder will return NOTE:\ \ link is not supported here.This method is often used when- user want to use\ \ a relative config path instead of hard-coding qlib config path in codeRaises------FileNotFoundError:If\ \ project path is not found" function: __init__.get_project_path - docstring: " \nThis function will init qlib automatically with following priority-\ \ Find the project configuration and init qlib- The parsing process will be affected\ \ by the `conf_type` of the configuration file- Init qlib with default config-\ \ Skip initialization if already initialized:**kwargs: it may contain following\ \ parameterscur_path: the start path to find the project pathHere are two examples\ \ of the configurationExample 1)If you want to create a new project-specific config\ \ based on a shared configure, you can use `conf_type: ref`.. code-block:: yamlconf_type:\ \ refqlib_cfg: '' # this could be null reference no\ \ config from other files# following configs in `qlib_cfg_update` is project=specificqlib_cfg_update:exp_manager:class:\ \ \"MLflowExpManager\"module_path: \"qlib.workflow.expm\"kwargs:uri: \"file://\"default_exp_name: \"Experiment\"Example 2)If you want\ \ to create simple a standalone config, you can use following config(a.k.a. `conf_type:\ \ origin`).. code-block:: pythonexp_manager:class: \"MLflowExpManager\"module_path:\ \ \"qlib.workflow.expm\"kwargs:uri: \"file://\"default_exp_name:\ \ \"Experiment\"" function: __init__.auto_init - docstring: null function: log.MetaLogger - docstring: " \nCustomized logger for Qlib." function: log.QlibLogger - docstring: null function: log.logger - docstring: null function: log.setLevel - docstring: null function: 'log._QLibLoggerManager:' - docstring: " \nGet a logger for a specific module.:param module_name: strLogic\ \ module name.:param level: int:return: LoggerLogger object." function: log.setLevel - docstring: null function: 'log.TimeInspector:' - docstring: " \nSet a time mark with current time, and this time mark will\ \ push into a stack.:return: floatA timestamp for current time." function: log.set_time_mark - docstring: " \nPop last time mark from stack." function: log.pop_time_mark - docstring: " \nGet last time mark from stack, calculate time diff with current\ \ time.:return: floatTime diff calculated by last time mark with current time." function: log.get_cost_time - docstring: " \nGet last time mark from stack, calculate time diff with current\ \ time, and log time diff and info.:param info: strInfo that will be logged into\ \ stdout." function: log.log_cost_time - docstring: ' logt. Log the time of the inside codeParameters----------name :nameshow_start :show_start' function: log.logt - docstring: ' set log with config :param log_config::return:' function: log.set_log_with_config - docstring: null function: log.LogFilter - docstring: null function: log.match_msg - docstring: null function: log.filter - docstring: ' set qlib.xxx logger handlers level Parameters----------level: intlogger levelreturn_orig_handler_level: boolreturn origin handler level mapExamples---------.. code-block:: pythonimport qlibimport loggingfrom qlib.log import get_module_logger, set_global_logger_levelqlib.init()tmp_logger_01 = get_module_logger("tmp_logger_01", level=logging.INFO)tmp_logger_01.info("1. tmp_logger_01 info show")global_level = logging.WARNING + 1set_global_logger_level(global_level)tmp_logger_02 = get_module_logger("tmp_logger_02", level=logging.INFO)tmp_logger_02.log(msg="2. tmp_logger_02 log show", level=global_level)tmp_logger_01.info("3. tmp_logger_01 info do not show")' function: log.set_global_logger_level - docstring: ' set qlib.xxx logger handlers level to use contextmanager Parameters----------level: intlogger levelExamples---------.. code-block:: pythonimport qlibimport loggingfrom qlib.log import get_module_logger, set_global_logger_level_cmqlib.init()tmp_logger_01 = get_module_logger("tmp_logger_01", level=logging.INFO)tmp_logger_01.info("1. tmp_logger_01 info show")global_level = logging.WARNING + 1with set_global_logger_level_cm(global_level):tmp_logger_02 = get_module_logger("tmp_logger_02", level=logging.INFO)tmp_logger_02.log(msg="2. tmp_logger_02 log show", level=global_level)tmp_logger_01.info("3. tmp_logger_01 info do not show")tmp_logger_01.info("4. tmp_logger_01 info show")' function: log.set_global_logger_level_cm - docstring: " \nParameters----------delete_zip_file : bool, optionalWhether\ \ to delete the zip file, value from True or False, by default False" function: 'data.GetData:' - docstring: null function: data.merge_remote_url - docstring: " \nDownload the specified file to the target folder.Parameters----------target_dir:\ \ strdata save directoryfile_name: strdataset name, needs to endwith .zip, value\ \ from [rl_data.zip, csv_data_cn.zip, ...]may contain folder names, for example:\ \ v2/qlib_data_simple_cn_1d_latest.zipdelete_old: booldelete an existing directory,\ \ by default TrueExamples---------# get rl datapython get_data.py download_data\ \ --file_name rl_data.zip --target_dir ~/.qlib/qlib_data/rl_dataWhen this command\ \ is run, the data will be downloaded from this link: https://qlibpublic.blob.core.windows.net/data/default/stock_data/rl_data.zip?{token}#\ \ get cn csv datapython get_data.py download_data --file_name csv_data_cn.zip\ \ --target_dir ~/.qlib/csv_data/cn_dataWhen this command is run, the data will\ \ be downloaded from this link: https://qlibpublic.blob.core.windows.net/data/default/stock_data/csv_data_cn.zip?{token}-------" function: data.download_data - docstring: null function: data.check_dataset - docstring: null function: data._unzip - docstring: null function: data._delete_qlib_data - docstring: ' download cn qlib data from remote Parameters----------target_dir: strdata save directoryname: strdataset name, value from [qlib_data, qlib_data_simple], by default qlib_dataversion: strdata version, value from [v1, ...], by default None(use script to specify version)interval: strdata freq, value from [1d], by default 1dregion: strdata region, value from [cn, us], by default cndelete_old: booldelete an existing directory, by default Trueexists_skip: boolexists skip, by default FalseExamples---------# get 1d datapython get_data.py qlib_data --name qlib_data --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cnWhen this command is run, the data will be downloaded from this link: https://qlibpublic.blob.core.windows.net/data/default/stock_data/v2/qlib_data_cn_1d_latest.zip?{token}# get 1min datapython get_data.py qlib_data --name qlib_data --target_dir ~/.qlib/qlib_data/cn_data_1min --interval 1min --region cnWhen this command is run, the data will be downloaded from this link: https://qlibpublic.blob.core.windows.net/data/default/stock_data/v2/qlib_data_cn_1min_latest.zip?{token}-------' function: data.qlib_data - docstring: null function: config.get_data_handler_config - docstring: null function: config.get_dataset_config - docstring: null function: config.get_gbdt_task - docstring: null function: config.get_record_lgb_config - docstring: null function: __init__.TestAutoData - docstring: null function: __init__.setUpClass - docstring: null function: __init__.TestOperatorData - docstring: ' MOCK_DF = pd.read_csv(io.StringIO(MOCK_DATA), header=0, dtype={"symbol": str})' function: __init__.setUpClass - docstring: null function: '__init__.MockStorageBase:' - docstring: null function: __init__.MockCalendarStorage - docstring: null function: __init__.data - docstring: null function: __init__.MockInstrumentStorage - docstring: null function: __init__.data - docstring: null function: __init__.MockFeatureStorage - docstring: null function: __init__.data - docstring: null function: __init__.start_index - docstring: null function: __init__.end_index - docstring: null function: __init__.TestMockData - docstring: ' Base strategy for trading self,outer_trade_decision: BaseTradeDecision = None,level_infra: LevelInfrastructure = None,common_infra: CommonInfrastructure = None,trade_exchange: Exchange = None,) -> None:' function: 'base.BaseStrategy:' - docstring: null function: base.executor - docstring: null function: base.trade_calendar - docstring: null function: base.trade_position - docstring: ' get trade exchange in a prioritized order return getattr(self, "_trade_exchange", None) or self.common_infra.get("trade_exchange")' function: base.trade_exchange - docstring: null function: base.reset_level_infra - docstring: null function: base.reset_common_infra - docstring: " \n- reset `level_infra`, used to reset trade calendar, .etc-\ \ reset `common_infra`, used to reset `trade_account`, `trade_exchange`, .etc-\ \ reset `outer_trade_decision`, used to make split decision**NOTE**:split this\ \ function into `reset` and `_reset` will make following cases more convenient1.\ \ Users want to initialize his strategy by overriding `reset`, but they don't\ \ want to affect the `_reset`called when initialization" function: base.reset - docstring: " \nPlease refer to the docs of `reset`" function: base._reset - docstring: ' Generate trade decision in each trading bar Parameters----------execute_result : List[object], optionalthe executed result for trade decision, by default None- When call the generate_trade_decision firstly, `execute_result` could be None' function: base.generate_trade_decision - docstring: " \nreturn data calendar's available decision range for `self`\ \ strategythe range consider following factors- data calendar in the charge of\ \ `self` strategy- trading range limitation from the decision of outer strategyrelated\ \ methods- TradeCalendarManager.get_data_cal_range- BaseTradeDecision.get_data_cal_range_limitParameters----------rtype:\ \ str- \"full\": return the available data index range of the strategy from `start_time`\ \ to `end_time`- \"step\": return the available data index range of the strategy\ \ of current stepReturns-------Tuple[int, int]:the available range both sides\ \ are closed" function: base.get_data_cal_avail_range - docstring: " \nupdate trade decision in each step of inner execution, this\ \ method enable all orderParameters----------trade_decision : BaseTradeDecisionthe\ \ trade decision that will be updatedtrade_calendar : TradeCalendarManagerThe\ \ calendar of the **inner strategy**!!!!!Returns-------BaseTradeDecision:" function: base.update_trade_decision - docstring: " \nA method for updating the outer_trade_decision.The outer strategy\ \ may change its decision during updating.Parameters----------outer_trade_decision\ \ : BaseTradeDecisionthe decision updated by the outer strategyReturns-------BaseTradeDecision" function: base.alter_outer_trade_decision - docstring: " \nA hook for doing sth after the upper level executor finished\ \ its execution (for example, finalizethe metrics collection)." function: base.post_upper_level_exe_step - docstring: " \nA hook for doing sth after the corresponding executor finished\ \ its execution.Parameters----------execute_result :the execution result" function: base.post_exe_step - docstring: ' RL-based strategy self,policy,outer_trade_decision: BaseTradeDecision = None,level_infra: LevelInfrastructure = None,common_infra: CommonInfrastructure = None,**kwargs,) -> None:' function: base.RLStrategy - docstring: ' (RL)-based (Strategy) with (Int)erpreter self,policy,state_interpreter: dict | StateInterpreter,action_interpreter: dict | ActionInterpreter,outer_trade_decision: BaseTradeDecision = None,level_infra: LevelInfrastructure = None,common_infra: CommonInfrastructure = None,**kwargs,) -> None:' function: base.RLIntStrategy - docstring: null function: 'objm.ObjManager:' - docstring: " \nsave obj as nameParameters----------obj : objectobject to\ \ be savedname : strname of the object" function: objm.save_obj - docstring: " \nsave objectsParameters----------obj_name_l : list of " function: objm.save_objs - docstring: " \nload object by nameParameters----------name : strthe name\ \ of the objectReturns-------object:loaded object" function: objm.load_obj - docstring: " \nif the object named `name` existsParameters----------name\ \ : strname of the objecTReturns-------bool:If the object exists" function: objm.exists - docstring: " \nlist the objectsReturns-------list:the list of returned objects" function: objm.list - docstring: ' remove. Parameters----------fname :if file name is provided. specific file is removedotherwise, The all the objects will be removed.' function: objm.remove - docstring: " \nUse file system to manage objects" function: objm.FileManager - docstring: null function: objm.create_path - docstring: null function: objm.save_obj - docstring: null function: objm.save_objs - docstring: null function: objm.load_obj - docstring: null function: objm.exists - docstring: null function: objm.list - docstring: " \nResample the calendar with frequency freq_raw into the calendar\ \ with frequency freq_samAssumption:- Fix length (240) of the calendar in each\ \ day.Parameters----------calendar_raw : np.ndarrayThe calendar with frequency\ \ freq_rawfreq_raw : strFrequency of the raw calendarfreq_sam : strSample frequencyregion:\ \ strRegion, for example, \"cn\", \"us\"Returns-------np.ndarrayThe calendar with\ \ frequency freq_sam" function: resam.resam_calendar - docstring: ' get the feature with higher or equal frequency than `freq`. Returns-------pd.DataFramethe feature with higher or equal frequency' function: resam.get_higher_eq_freq_feature - docstring: " \nResample value from time-series data- If `feature` has MultiIndex[instrument,\ \ datetime], apply the `method` to each instruemnt data with datetime in [start_time,\ \ end_time]Example:.. code-block::print(feature)$close $volumeinstrument\ \ datetimeSH600000 2010-01-04 86.778313 16162960.02010-01-05 87.433578\ \ 28117442.02010-01-06 85.713585 23632884.02010-01-07 83.788803 20813402.02010-01-08\ \ 84.730675 16044853.0SH600655 2010-01-04 2699.567383 158193.3281252010-01-08\ \ 2612.359619 77501.4062502010-01-11 2712.982422 160852.3906252010-01-12\ \ 2788.688232 164587.9375002010-01-13 2790.604004 145460.453125print(resam_ts_data(feature,\ \ start_time=\"2010-01-04\", end_time=\"2010-01-05\", fields=[\"$close\", \"$volume\"\ ], method=\"last\"))$close $volumeinstrumentSH600000 87.433578 28117442.0SH600655\ \ 2699.567383 158193.328125- Else, the `feature` should have Index[datetime],\ \ just apply the `method` to `feature` directlyExample:.. code-block::print(feature)$close\ \ $volumedatetime2010-01-04 86.778313 16162960.02010-01-05 87.433578\ \ 28117442.02010-01-06 85.713585 23632884.02010-01-07 83.788803 20813402.02010-01-08\ \ 84.730675 16044853.0print(resam_ts_data(feature, start_time=\"2010-01-04\"\ , end_time=\"2010-01-05\", method=\"last\"))$close 87.433578$volume 28117442.0print(resam_ts_data(feature['$close'],\ \ start_time=\"2010-01-04\", end_time=\"2010-01-05\", method=\"last\"))87.433578Parameters----------ts_feature\ \ : Union[pd.DataFrame, pd.Series]Raw time-series feature to be resampledstart_time\ \ : Union[str, pd.Timestamp], optionalstart sampling time, by default Noneend_time\ \ : Union[str, pd.Timestamp], optionalend sampling time, by default Nonemethod\ \ : Union[str, Callable], optionalsample method, apply method function to each\ \ stock series data, by default \"last\"- If type(method) is str or callable function,\ \ it should be an attribute of SeriesGroupBy or DataFrameGroupby, and applies\ \ groupy.method for the sliced time-series data- If method is None, do nothing\ \ for the sliced time-series data.method_kwargs : dict, optionalarguments of method,\ \ by default {}Returns-------The resampled DataFrame/Series/value, return None\ \ when the resampled data is empty." function: resam.resam_ts_data - docstring: ' get the first/last not nan value of pd.Series with single level index Parameters----------series : pd.Seriesseries should not be emptylast : bool, optionalwhether to get the last valid value, by default True- if last is True, get the last valid value- else, get the first valid valueReturns-------Nan | floatthe first/last valid value' function: resam.get_valid_value - docstring: ' Robust ZScore Normalization Use robust statistics for Z-Score normalization:mean(x) = median(x)std(x) = MAD(x) * 1.4826Reference:https://en.wikipedia.org/wiki/Median_absolute_deviation.' function: data.robust_zscore - docstring: null function: data.zscore - docstring: " \ndeepcopy an object without copy the complicated objects.This is\ \ useful when you want to generate Qlib tasks and share the handlerNOTE:- This\ \ function can't handle recursive objects!!!!!Parameters----------obj : objectthe\ \ object to be copiedReturns-------object:The copied object" function: data.deepcopy_basic_type - docstring: " \nsupporting adding base config based on the ext_config>>> bc =\ \ {\"a\": \"xixi\"}>>> ec = {\"b\": \"haha\"}>>> new_bc = update_config(bc, ec)>>>\ \ print(new_bc){'a': 'xixi', 'b': 'haha'}>>> print(bc) # base config should not\ \ be changed{'a': 'xixi'}>>> print(update_config(bc, {\"b\": S_DROP})){'a': 'xixi'}>>>\ \ print(update_config(new_bc, {\"b\": S_DROP})){'a': 'xixi'}" function: data.update_config - docstring: ' Create or get a file or directory given the path and return_dir. Parameters----------path: a string indicates the path or None indicates creating a temporary path.return_dir: if True, create and return a directory; otherwise c&r a file.' function: file.get_or_create_path - docstring: ' Save multiple parts file Implementation process:1. get the absolute path to ''filename''2. create a ''filename'' directory3. user does something with file_path(''filename/'')4. remove ''filename'' directory5. make_archive ''filename'' directory, and rename ''archive file'' to filename:param filename: result model path:param format: archive format: one of "zip", "tar", "gztar", "bztar", or "xztar":return: real model pathUsage::>>> # The following code will create an archive file(''~/tmp/test_file'') containing ''test_doc_i''(i is 0-10) files.>>> with save_multiple_parts_file(''~/tmp/test_file'') as filename_dir:... for i in range(10):... temp_path = os.path.join(filename_dir, ''test_doc_{}''.format(str(i)))... with open(temp_path) as fp:... fp.write(str(i))...' function: file.save_multiple_parts_file - docstring: ' Unpack archive with archive buffer After the call is finished, the archive file and directory will be deleted.Implementation process:1. create ''tempfile'' in ''~/tmp/'' and directory2. ''buffer'' write to ''tempfile''3. unpack archive file(''tempfile'')4. user does something with file_path(''tempfile/'')5. remove ''tempfile'' and ''tempfile directory'':param buffer: bytes:param format: archive format: one of "zip", "tar", "gztar", "bztar", or "xztar":return: unpack archive directory pathUsage::>>> # The following code is to print all the file names in ''test_unpack.tar.gz''>>> with open(''test_unpack.tar.gz'') as fp:... buffer = fp.read()...>>> with unpack_archive_with_buffer(buffer) as temp_dir:... for f_n in os.listdir(temp_dir):... print(f_n)...' function: file.unpack_archive_with_buffer - docstring: null function: file.get_tmp_file_with_buffer - docstring: " \nproviding a easy interface to get an IO objectParameters----------file\ \ : Union[IO, str, Path]a object representing the fileReturns-------IO:a IO-like\ \ objectRaises------NotImplementedError:" function: file.get_io_object - docstring: null function: paral.ParallelExt - docstring: ' datetime_groupby_apply This function will apply the `apply_func` on the datetime level index.Parameters----------df :DataFrame for processingapply_func : Union[Callable, Text]apply_func for processing the dataif a string is given, then it is treated as naive pandas functionaxis :which axis is the datetime level locatedlevel :which level is the datetime levelresample_rule :How to resample the data to calculating paralleln_jobs :n_jobs for joblibReturns:pd.DataFrame' function: paral.datetime_groupby_apply - docstring: null function: paral._naive_group_apply - docstring: " \nThis AsyncCaller tries to make it easier to async callCurrently,\ \ it is used in MLflowRecorder to make functions like `log_params` asyncNOTE:-\ \ This caller didn't consider the return value" function: 'paral.AsyncCaller:' - docstring: null function: paral.close - docstring: null function: paral.run - docstring: null function: paral.wait - docstring: null function: paral.async_dec - docstring: null function: paral.decorator_func - docstring: null function: paral.wrapper - docstring: null function: 'paral.DelayedTask:' - docstring: ' get_delayed_tuple. Return the delayed_tuple created by joblib.delayed' function: paral.get_delayed_tuple - docstring: ' set_res. Parameters----------res :the executed result of the delayed tuple' function: paral.set_res - docstring: ' return the object to replace the delayed task raise NotImplementedError("NotImplemented")' function: paral.get_replacement - docstring: null function: paral.DelayedTuple - docstring: null function: paral.get_delayed_tuple - docstring: null function: paral.get_replacement - docstring: ' DelayedDict. It is designed for following feature:Converting following existing code to parallel- constructing a dict- key can be gotten instantly- computation of values tasks a lot of time.- AND ALL the values are calculated in a SINGLE function' function: paral.DelayedDict - docstring: null function: paral.get_delayed_tuple - docstring: null function: paral.get_replacement - docstring: ' is_delayed_tuple. Parameters----------obj : objectReturns-------boolis `obj` joblib.delayed tuple' function: paral.is_delayed_tuple - docstring: ' _replace_and_get_dt. FIXME: this function may cause infinite loop when the complex data-structure contains loop-referenceParameters----------complex_iter :complex_iter' function: paral._replace_and_get_dt - docstring: ' _recover_dt. replace all the DelayedTask in the `complex_iter` with its `.res` valueFIXME: this function may cause infinite loop when the complex data-structure contains loop-referenceParameters----------complex_iter :complex_iter' function: paral._recover_dt - docstring: ' complex_parallel. Find all the delayed function created by delayed in complex_iter, run them parallelly and then replace it with the result>>> from qlib.utils.paral import complex_parallel>>> from joblib import Parallel, delayed>>> complex_iter = {"a": delayed(sum)([1,2,3]), "b": [1, 2, delayed(sum)([10, 1])]}>>> complex_parallel(Parallel(), complex_iter){''a'': 6, ''b'': [1, 2, 11]}Parameters----------paral : Parallelparalcomplex_iter :NOTE: only list, tuple and dict will be explored!!!!Returns-------complex_iter whose delayed joblib tasks are replaced with its execution results.' function: paral.complex_parallel - docstring: " \nWhen we repeatedly run functions, it is hard to avoid memory leakage.So\ \ we run it in the subprocess to ensure it is OK.NOTE: Because local object can't\ \ be pickled. So we can't implement it via closure.We have to implement it via\ \ callable Class" function: 'paral.call_in_subproc:' - docstring: null function: exceptions.QlibException - docstring: ' Error type for re-initialization when starting an experiment ' function: exceptions.RecorderInitializationError - docstring: ' Error type for Recorder when can not load object ' function: exceptions.LoadObjectError - docstring: ' Load module path :param module_path::return::raises: ModuleNotFoundError' function: mod.get_module_by_module_path - docstring: " \nParameters----------module_path : stre.g. \"a.b.c.ClassName\"\ Returns-------Tuple[str, str]e.g. (\"a.b.c\", \"ClassName\")" function: mod.split_module_path - docstring: " \nextract class/func and kwargs from config infoParameters----------config\ \ : [dict, str]similar to configplease refer to the doc of init_instance_by_configdefault_module\ \ : Python module or strIt should be a python module to load the class typeThis\ \ function will load class from the config['module_path'] first.If config['module_path']\ \ doesn't exists, it will load the class from default_module.Returns-------(type,\ \ dict):the class/func object and it's arguments.Raises------ModuleNotFoundError" function: mod.get_callable_kwargs - docstring: " \nget initialized instance with configParameters----------config\ \ : InstConfdefault_module : Python moduleOptional. It should be a python module.NOTE:\ \ the \"module_path\" will be override by `module` argumentsThis function will\ \ load class from the config['module_path'] first.If config['module_path'] doesn't\ \ exists, it will load the class from default_module.accept_types: Union[type,\ \ Tuple[type]]Optional. If the config is a instance of specific type, return the\ \ config directly.This will be passed into the second parameter of isinstance.try_kwargs:\ \ DictTry to pass in kwargs in `try_kwargs` when initialized the instanceIf error\ \ occurred, it will fail back to initialization without try_kwargs.Returns-------object:An\ \ initialized object based on the config info" function: mod.init_instance_by_config - docstring: " \nPython doesn't provide the downcasting mechanism.We use the trick\ \ here to downcast the classParameters----------obj : objectthe object to be castcls\ \ : typethe target class type" function: mod.class_casting - docstring: " \nFind all the classes recursively that inherit from `cls` in a\ \ given module.- `cls` itself is also included>>> from qlib.data.dataset.handler\ \ import DataHandler>>> find_all_classes(\"qlib.contrib.data.handler\", DataHandler)[, ,\ \ , ,\ \ ]>>> from qlib.contrib.rolling.base\ \ import Rolling>>> find_all_classes(\"qlib.contrib.rolling\", Rolling)[, ]TODO:-\ \ skip import error" function: mod.find_all_classes - docstring: " \nSerializable will change the behaviors of pickle.The rule to tell\ \ if a attribute will be kept or dropped when dumping.The rule with higher priorities\ \ is on the top- in the config attribute list -> always dropped- in the include\ \ attribute list -> always kept- in the exclude attribute list -> always dropped-\ \ name not starts with `_` -> kept- name starts with `_` -> kept if `dump_all`\ \ is true else droppedIt provides a syntactic sugar for distinguish the attributes\ \ which user doesn't want.- For examples, a learnable Datahandler just wants to\ \ save the parameters without data when dumping to disk" function: 'serial.Serializable:' - docstring: null function: serial._is_kept - docstring: " \nwill the object dump all object" function: serial.dump_all - docstring: " \nWhat attribute will not be in specific listParameters----------attr_type\ \ : str\"include\" or \"exclude\"Returns-------list:" function: serial._get_attr_list - docstring: " \nconfigure the serializable objectParameters----------kwargs\ \ may include following keysdump_all : boolwill the object dump all objectexclude\ \ : listWhat attribute will not be dumpedinclude : listWhat attribute will be\ \ dumpedrecursive : boolwill the configuration be recursive" function: serial.config - docstring: " \nDump self to a pickle file.path (Union[Path, str]): the path\ \ to dumpkwargs may include following keysdump_all : boolwill the object dump\ \ all objectexclude : listWhat attribute will not be dumpedinclude : listWhat\ \ attribute will be dumped" function: serial.to_pickle - docstring: " \nLoad the serializable class from a filepath.Args:filepath\ \ (str): the path of fileRaises:TypeError: the pickled file must be `type(cls)`Returns:`type(cls)`:\ \ the instance of `type(cls)`" function: serial.load - docstring: " \nReturn the real backend of a Serializable class. The pickle_backend\ \ value can be \"pickle\" or \"dill\".Returns:module: pickle or dill module based\ \ on pickle_backend" function: serial.get_backend - docstring: " \nA general dumping method for objectParameters----------obj\ \ : objectthe object to be dumpedpath : Union[Path, str]the target path the data\ \ will be dumped" function: serial.general_dump - docstring: ' concat all SingleData by index. TODO: now just for SingleData.Parameters----------data_list : List[SingleData]the list of all SingleData to concat.Returns-------MultiDatathe MultiData with ndim == 2' function: index_data.concat - docstring: ' concat all SingleData by new index. Parameters----------data_list : List[SingleData]the list of all SingleData to sum.new_index : listthe new_index of new SingleData.fill_value : floatfill the missing values or replace np.NaN.Returns-------SingleDatathe SingleData with new_index and values after sum.' function: index_data.sum_by_index - docstring: " \nThis is for indexing(rows or columns)Read-only operations has\ \ higher priorities than others.So this class is designed in a **read-only** way\ \ to shared data for queries.Modifications will results in new Index.NOTE: the\ \ indexing has following flaws- duplicated index value is not well supported (only\ \ the first appearance will be considered)- The order of the index is not considered!!!!\ \ So the slicing will not behave like pandas when indexings are ordered" function: 'index_data.Index:' - docstring: " \nAfter user creates indices with Type A, user may query data\ \ with other types with the same info.This method try to make type conversion\ \ and make query sane rather than raising KeyError strictlyParameters----------item\ \ :The item to query index" function: index_data._convert_type - docstring: " \nGiven the index value, get the integer indexParameters----------item\ \ :The item to queryReturns-------int:The index of the itemRaises------KeyError:If\ \ the query item does not exist" function: index_data.index - docstring: null function: index_data.is_sorted - docstring: " \nsort the indexReturns-------Tuple[\"Index\", np.ndarray]:the\ \ sorted Index and the changed index" function: index_data.sort - docstring: ' return the index with the format of list. return self.idx_list.tolist()' function: index_data.tolist - docstring: " \n`Indexer` will behave like the `LocIndexer` in PandasRead-only\ \ operations has higher priorities than others.So this class is designed in a\ \ read-only way to shared data for queries.Modifications will results in new Index." function: 'index_data.LocIndexer:' - docstring: ' process the indices from user and output a list of `Index` res = []for i, idx in enumerate(indices):res.append(Index(data_shape[i] if len(idx) == 0 else idx))return res' function: index_data.proc_idx_l - docstring: " \nconvert value-based indexing to integer-based indexing.Parameters----------index\ \ : Indexindex data.indexing : slicevalue based indexing data with slice type\ \ for indexing.Returns-------slice:the integer based slicing" function: index_data._slc_convert - docstring: null function: 'index_data.BinaryOps:' - docstring: " \nmeta class for auto generating operations for index data." function: index_data.index_data_ops_creator - docstring: " \nBase data structure of SingleData and MultiData.NOTE:- For performance\ \ issue, only **np.floating** is supported in the underlayer data !!!- Boolean\ \ based on np.floating is also supported. Here are some examples.. code-block::\ \ pythonnp.array([ np.nan]).any() -> Truenp.array([ np.nan]).all() -> Truenp.array([1.\ \ , 0.]).any() -> Truenp.array([1. , 0.]).all() -> False" function: index_data.IndexData - docstring: null function: index_data.loc - docstring: null function: index_data.iloc - docstring: null function: index_data.index - docstring: null function: index_data.columns - docstring: " \nAlign all indices of `other` to `self` before performing the\ \ arithmetic operations.This function will return a new IndexData rather than\ \ changing data in `other` inplaceParameters----------other : \"IndexData\"the\ \ index in `other` is to be changedReturns-------IndexData:the data in `other`\ \ with index aligned to `self`" function: index_data._align_indices - docstring: null function: index_data.sort_index - docstring: ' get the abs of data except np.NaN. tmp_data = np.absolute(self.data)return self.__class__(tmp_data, *self.indices)' function: index_data.abs - docstring: null function: index_data.replace - docstring: ' apply a function to data. tmp_data = func(self.data)return self.__class__(tmp_data, *self.indices)the length of the data.Returns-------intthe length of the data.' function: index_data.apply - docstring: null function: index_data.sum - docstring: null function: index_data.mean - docstring: null function: index_data.isna - docstring: null function: index_data.fillna - docstring: null function: index_data.count - docstring: null function: index_data.all - docstring: null function: index_data.empty - docstring: null function: index_data.values - docstring: ' A data structure of index and numpy data. It''s used to replace pd.Series due to high-speed.Parameters----------data : Union[int, float, np.number, list, dict, pd.Series]the input dataindex : Union[list, pd.Index]the index of data.empty list indicates that auto filling the index to the length of data' function: index_data.SingleData - docstring: null function: index_data._align_indices - docstring: ' reindex data and fill the missing value with np.NaN. Parameters----------new_index : listnew indexfill_value:what value to fill if index is missingReturns-------SingleDatareindex data' function: index_data.reindex - docstring: null function: index_data.add - docstring: ' convert SingleData to dict. Returns-------dictdata with the dict format.' function: index_data.to_dict - docstring: null function: index_data.to_series - docstring: ' A data structure of index and numpy data. It''s used to replace pd.DataFrame due to high-speed.Parameters----------data : Union[list, np.ndarray]the dim of data must be 2.index : Union[List, pd.Index, Index]the index of data.columns: Union[List, pd.Index, Index]the columns of data.' function: index_data.MultiData - docstring: " \nget the minute level calendar in day periodParameters----------shift\ \ : intthe shift direction would be like pandas shift.series.shift(1) will replace\ \ the value at `i`-th with the one at `i-1`-thregion: strRegion, for example,\ \ \"cn\", \"us\"Returns-------List[time]:" function: time.get_min_cal - docstring: ' Is there only one piece of data for stock market. Parameters----------start_time : Union[pd.Timestamp, str]closed start time for data.end_time : Union[pd.Timestamp, str]closed end time for data.freq :region: strRegion, for example, "cn", "us"Returns-------boolTrue means one piece of data to obtain.' function: time.is_single_value - docstring: null function: 'time.Freq:' - docstring: " \nParse freq into a unified formatParameters----------freq :\ \ strRaw freq, supported freq should match the re '^([0-9]*)(month|mon|week|w|day|d|minute|min)$'Returns-------freq:\ \ Tuple[int, str]Unified freq, including freq count and unified freq unit. The\ \ freq unit should be '[month|week|day|minute]'.Example:.. code-block::print(Freq.parse(\"\ day\"))(1, \"day\" )print(Freq.parse(\"2mon\"))(2, \"month\")print(Freq.parse(\"\ 10w\"))(10, \"week\")" function: time.parse - docstring: " \nget pd.Timedeta objectParameters----------n : intfreq : strTypically,\ \ they are the return value of Freq.parseReturns-------pd.Timedelta:" function: time.get_timedelta - docstring: ' Calculate freq delta Parameters----------left_frq: strright_freq: strReturns-------' function: time.get_min_delta - docstring: ' Get the closest freq to base_freq from freq_list Parameters----------base_freqfreq_listReturns-------if the recent frequency is foundFreqelse:None' function: time.get_recent_freq - docstring: null function: time.time_to_day_index - docstring: " \nget the min-bar index in a day for a time range (both left and\ \ right is closed) given a fixed frequencyParameters----------start : stre.g.\ \ \"9:30\"end : stre.g. \"14:30\"freq : str\"1min\"Returns-------Tuple[int, int]:The\ \ index of start and end in the calendar. Both left and right are **closed**" function: time.get_day_min_idx_range - docstring: null function: time.concat_date_time - docstring: " \nalign the minute-level data to a down sampled calendare.g. align\ \ 10:38 to 10:35 in 5 minute-level(10:30 in 10 minute-level)Parameters----------x\ \ : pd.Timestampdatetime to be alignedsam_minutes : intalign to `sam_minutes`\ \ minute-level calendarregion: strRegion, for example, \"cn\", \"us\"Returns-------pd.Timestamp:the\ \ datetime after aligned" function: time.cal_sam_minute - docstring: " \nchange the time by infinitely small quantity.Parameters----------date_time\ \ : pd.Timestampthe original timedirection : strthe direction the time are going\ \ to- \"backward\" for going to history- \"forward\" for going to the futureReturns-------pd.Timestamp:the\ \ shifted time" function: time.epsilon_change - docstring: ' get redis connection instance. return redis.StrictRedis(host=C.redis_host, port=C.redis_port, db=C.redis_task_db, password=C.redis_password)#################### Data ####################' function: __init__.get_redis_connection - docstring: null function: __init__.read_bin - docstring: " \nThis method will be used in PIT database.It return all the possible\ \ values between `first` and `end` (first and end is included)Parameters----------quarterly\ \ : boolwill it return quarterly index or yearly index.Returns-------List[int]the\ \ possible index between [first, last]" function: __init__.get_period_list - docstring: null function: __init__.get_period_offset - docstring: " \nAt `cur_date`(e.g. 20190102), read the information at `period`(e.g.\ \ 201803).Only the updating info before cur_date or at cur_date will be used.Parameters----------period:\ \ intdate period represented by interger, e.g. 201901 corresponds to the first\ \ quarter in 2019cur_date_int: intdate which represented by interger, e.g. 20190102last_period_index:\ \ intit is a optional parameter; it is designed to avoid repeatedly access the\ \ .index data of PIT database whensequentially observing the data (Because the\ \ latest index of a specific period of data certainly appear in after the one\ \ in last observation).Returns-------the query value and byte index the index\ \ value" function: __init__.read_period_data - docstring: " \nforward fill a 1D numpy arrayParameters----------arr : np.arrayInput\ \ numpy 1D array" function: __init__.np_ffill - docstring: ' multi fields list lower bound. for single field list use `bisect.bisect_left` instead' function: __init__.lower_bound - docstring: ' multi fields list upper bound. for single field list use `bisect.bisect_right` instead' function: __init__.upper_bound - docstring: null function: __init__.requests_with_retry - docstring: null function: __init__.parse_config - docstring: null function: __init__.drop_nan_by_y_index - docstring: null function: __init__.hash_args - docstring: null function: __init__.parse_field - docstring: ' Compare dict value :param src_data::param dst_data::return:' function: __init__.compare_dict_value - docstring: null function: __init__.DateEncoder - docstring: null function: __init__.default - docstring: ' remove repeat field :param fields: list; features fields:return: list' function: __init__.remove_repeat_field - docstring: ' remove fields space :param fields: features fields:return: list or str' function: __init__.remove_fields_space - docstring: ' normalize cache fields :param fields: features fields:return: list' function: __init__.normalize_cache_fields - docstring: ' normalize cache instruments :return: list or dict' function: __init__.normalize_cache_instruments - docstring: ' judgy whether date is a tradable date ----------date : pandas.Timestampcurrent date' function: __init__.is_tradable_date - docstring: ' get trading date range by shift Parameters----------trading_date: pd.Timestampleft_shift: intright_shift: intfuture: bool' function: __init__.get_date_range - docstring: ' get trading date with shift bias will cur_date e.g. : shift == 1, return next trading dateshift == -1, return previous trading date----------trading_date : pandas.Timestampcurrent dateshift : intclip_shift: boolalign : Optional[str]When align is None, this function will raise ValueError if `trading_date` is not a trading datewhen align is "left"/"right", it will try to align to left/right nearest trading date before shifting when `trading_date` is not a trading date' function: __init__.get_date_by_shift - docstring: ' get next trading date ----------cur_date : pandas.Timestampcurrent date' function: __init__.get_next_trading_date - docstring: ' get previous trading date ----------date : pandas.Timestampcurrent date' function: __init__.get_pre_trading_date - docstring: ' handle the end date with various format If end_date is -1, None, or end_date is greater than the maximum trading day, the last trading date is returned.Otherwise, returns the end_date----------end_date: strend trading datedate : pandas.Timestampcurrent date' function: __init__.transform_end_date - docstring: ' Get the date(YYYY-MM-DD) written in file name Parameterfile_name : str:returndate : str''YYYY-MM-DD''' function: __init__.get_date_in_file_name - docstring: ' split the score file into two part Parameter---------pred : pd.DataFrame (index:)A score file of stocksnumber: the number of dates for pred_leftsplit_date: the last date of the pred_leftReturn-------pred_left : pd.DataFrame (index:)The first part of original score filepred_right : pd.DataFrame (index:)The second part of original score file' function: __init__.split_pred - docstring: " \nTime slicing in Qlib or Pandas is a frequently-used action.However,\ \ user often input all kinds of data format to represent time.This function will\ \ help user to convert these inputs into a uniform format which is friendly to\ \ time slicing.Parameters----------t : Union[None, str, pd.Timestamp]original\ \ timeReturns-------Union[None, pd.Timestamp]:" function: __init__.time_to_slc_point - docstring: null function: __init__.can_use_cache - docstring: null function: __init__.exists_qlib_data - docstring: null function: __init__.check_qlib_data - docstring: " \nmake the df index sorteddf.sort_index() will take a lot of time\ \ even when `df.is_lexsorted() == True`This function could avoid such caseParameters----------df\ \ : pd.DataFrameReturns-------pd.DataFrame:sorted dataframe" function: __init__.lazy_sort_index - docstring: " \nFlatten a nested dict.>>> flatten_dict({'a': 1, 'c': {'a': 2,\ \ 'b': {'x': 5, 'y' : 10}}, 'd': [1, 2, 3]})>>> {'a': 1, 'c.a': 2, 'c.b.x': 5,\ \ 'd': [1, 2, 3], 'c.b.y': 10}>>> flatten_dict({'a': 1, 'c': {'a': 2, 'b': {'x':\ \ 5, 'y' : 10}}, 'd': [1, 2, 3]}, sep=FLATTEN_TUPLE)>>> {'a': 1, ('c','a'): 2,\ \ ('c','b','x'): 5, 'd': [1, 2, 3], ('c','b','y'): 10}Args:d (dict): the dict\ \ waiting for flattingparent_key (str, optional): the parent key, will be a prefix\ \ in new key. Defaults to \"\".sep (str, optional): the separator for string connecting.\ \ FLATTEN_TUPLE for tuple connecting.Returns:dict: flatten dict" function: __init__.flatten_dict - docstring: " \nFollow the name_path to get values from configFor example:If we\ \ follow the example in in the Parameters section,Timestamp('2008-01-02 00:00:00')\ \ will be returnedParameters----------config : dicte.g.{'dataset': {'class': 'DatasetH','kwargs':\ \ {'handler': {'class': 'Alpha158','kwargs': {'end_time': '2020-08-01','fit_end_time':\ \ '','fit_start_time': '','instruments':\ \ 'csi100','start_time': '2008-01-01'},'module_path': 'qlib.contrib.data.handler'},'segments':\ \ {'test': (Timestamp('2017-01-03 00:00:00'),Timestamp('2019-04-08 00:00:00')),'train':\ \ (Timestamp('2008-01-02 00:00:00'),Timestamp('2014-12-31 00:00:00')),'valid':\ \ (Timestamp('2015-01-05 00:00:00'),Timestamp('2016-12-30 00:00:00'))}}}}name_path\ \ : stre.g.\"dataset.kwargs.segments.train.1\"Returns-------objectthe retrieved\ \ object" function: __init__.get_item_from_obj - docstring: " \nDetect placeholder in config and fill them with config_extend.The\ \ item of dict must be single item(int, str, etc), dict and list. Tuples are not\ \ supported.There are two type of variables:- user-defined variables :e.g. when\ \ config_extend is `{\"\": model, \"\": dataset}`, \"\"\ \ and \"\" in `config` will be replaced with `model` `dataset`- variables\ \ extracted from `config` :e.g. the variables like \"\"\ \ will be replaced with the values from `config`Parameters----------config : dictthe\ \ parameter dict will be filledconfig_extend : dictthe value of all placeholdersReturns-------dictthe\ \ parameter dict" function: __init__.fill_placeholder - docstring: null function: __init__.try_replace_placeholder - docstring: " \nthis will work like a decoration functionThe decrated function\ \ will ignore and give warning when the parameter is not acceptableFor example,\ \ if you have a function `f` which may optionally consume the keywards `bar`.then\ \ you can call it by `auto_filter_kwargs(f)(bar=3)`, which will automatically\ \ filter out`bar` when f does not need barParameters----------func : CallableThe\ \ original functionReturns-------Callable:the new callable function" function: __init__.auto_filter_kwargs - docstring: null function: __init__._func - docstring: ' Wrapper class for anything that needs to set up during qlib.init self._provider = None' function: '__init__.Wrapper:' - docstring: null function: __init__.register - docstring: ' register_wrapper :param wrapper: A wrapper.:param cls_or_obj: A class or class name or object instance.' function: __init__.register_wrapper - docstring: ' load dataset from multiple file formats if isinstance(path_or_obj, pd.DataFrame):return path_or_objif not os.path.exists(path_or_obj):raise ValueError(f"file {path_or_obj} doesn''t exist")_, extension = os.path.splitext(path_or_obj)if extension == ".h5":return pd.read_hdf(path_or_obj)elif extension == ".pkl":return pd.read_pickle(path_or_obj)elif extension == ".csv":return pd.read_csv(path_or_obj, parse_dates=True, index_col=index_col)raise ValueError(f"unsupported file type `{extension}`")' function: __init__.load_dataset - docstring: ' stock code to file name Parameters----------code: str' function: __init__.code_to_fname - docstring: ' file name to stock code Parameters----------fname: str' function: __init__.fname_to_code - docstring: ' Element-wise Operator Parameters----------feature : Expressionfeature instanceReturns----------Expressionfeature operation output' function: ops.ElemOperator - docstring: null function: ops.get_longest_back_rolling - docstring: null function: ops.get_extended_window_size - docstring: ' Change Instrument Operator In some case, one may want to change to another instrument when calculating, for example, tocalculate beta of a stock with respect to a market index.This would require changing the calculation of features from the stock (original instrument) tothe index (reference instrument)Parameters----------instrument: new instrument for which the downstream operations should be performed upon.i.e., SH000300 (CSI300 index), or ^GPSC (SP500 index).feature: the feature to be calculated for the new instrument.Returns----------Expressionfeature operation output' function: ops.ChangeInstrument - docstring: null function: ops.load - docstring: null function: ops._load_internal - docstring: ' Numpy Element-wise Operator Parameters----------feature : Expressionfeature instancefunc : strnumpy feature operation methodReturns----------Expressionfeature operation output' function: ops.NpElemOperator - docstring: null function: ops._load_internal - docstring: ' Feature Absolute Value Parameters----------feature : Expressionfeature instanceReturns----------Expressiona feature instance with absolute output' function: ops.Abs - docstring: ' Feature Sign Parameters----------feature : Expressionfeature instanceReturns----------Expressiona feature instance with sign' function: ops.Sign - docstring: " \nTo avoid error raised by bool type input, we transform the\ \ data into float32." function: ops._load_internal - docstring: ' Feature Log Parameters----------feature : Expressionfeature instanceReturns----------Expressiona feature instance with log' function: ops.Log - docstring: ' Feature Mask Parameters----------feature : Expressionfeature instanceinstrument : strinstrument maskReturns----------Expressiona feature instance with masked instrument' function: ops.Mask - docstring: null function: ops._load_internal - docstring: ' Not Operator Parameters----------feature : Expressionfeature instanceReturns----------Feature:feature elementwise not output' function: ops.Not - docstring: ' Pair-wise operator Parameters----------feature_left : Expressionfeature instance or numeric valuefeature_right : Expressionfeature instance or numeric valueReturns----------Feature:two features'' operation output' function: ops.PairOperator - docstring: null function: ops.get_longest_back_rolling - docstring: null function: ops.get_extended_window_size - docstring: ' Numpy Pair-wise operator Parameters----------feature_left : Expressionfeature instance or numeric valuefeature_right : Expressionfeature instance or numeric valuefunc : stroperator functionReturns----------Feature:two features'' operation output' function: ops.NpPairOperator - docstring: null function: ops._load_internal - docstring: ' Power Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceReturns----------Feature:The bases in feature_left raised to the exponents in feature_right' function: ops.Power - docstring: ' Add Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceReturns----------Feature:two features'' sum' function: ops.Add - docstring: ' Subtract Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceReturns----------Feature:two features'' subtraction' function: ops.Sub - docstring: ' Multiply Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceReturns----------Feature:two features'' product' function: ops.Mul - docstring: ' Division Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceReturns----------Feature:two features'' division' function: ops.Div - docstring: ' Greater Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceReturns----------Feature:greater elements taken from the input two features' function: ops.Greater - docstring: ' Less Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceReturns----------Feature:smaller elements taken from the input two features' function: ops.Less - docstring: ' Greater Than Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceReturns----------Feature:bool series indicate `left > right`' function: ops.Gt - docstring: ' Greater Equal Than Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceReturns----------Feature:bool series indicate `left >= right`' function: ops.Ge - docstring: ' Less Than Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceReturns----------Feature:bool series indicate `left < right`' function: ops.Lt - docstring: ' Less Equal Than Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceReturns----------Feature:bool series indicate `left <= right`' function: ops.Le - docstring: ' Equal Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceReturns----------Feature:bool series indicate `left == right`' function: ops.Eq - docstring: ' Not Equal Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceReturns----------Feature:bool series indicate `left != right`' function: ops.Ne - docstring: ' And Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceReturns----------Feature:two features'' row by row & output' function: ops.And - docstring: ' Or Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceReturns----------Feature:two features'' row by row | outputs' function: ops.Or - docstring: ' If Operator Parameters----------condition : Expressionfeature instance with bool values as conditionfeature_left : Expressionfeature instancefeature_right : Expressionfeature instance' function: ops.If - docstring: null function: ops._load_internal - docstring: null function: ops.get_longest_back_rolling - docstring: null function: ops.get_extended_window_size - docstring: ' Rolling Operator The meaning of rolling and expanding is the same in pandas.When the window is set to 0, the behaviour of the operator should follow `expanding`Otherwise, it follows `rolling`Parameters----------feature : Expressionfeature instanceN : introlling window sizefunc : strrolling methodReturns----------Expressionrolling outputs' function: ops.Rolling - docstring: null function: ops._load_internal - docstring: null function: ops.get_longest_back_rolling - docstring: null function: ops.get_extended_window_size - docstring: ' Feature Reference Parameters----------feature : Expressionfeature instanceN : intN = 0, retrieve the first data; N > 0, retrieve data of N periods ago; N < 0, future dataReturns----------Expressiona feature instance with target reference' function: ops.Ref - docstring: null function: ops._load_internal - docstring: null function: ops.get_longest_back_rolling - docstring: null function: ops.get_extended_window_size - docstring: ' Rolling Mean (MA) Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling average' function: ops.Mean - docstring: ' Rolling Sum Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling sum' function: ops.Sum - docstring: ' Rolling Std Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling std' function: ops.Std - docstring: ' Rolling Variance Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling variance' function: ops.Var - docstring: ' Rolling Skewness Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling skewness' function: ops.Skew - docstring: ' Rolling Kurtosis Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling kurtosis' function: ops.Kurt - docstring: ' Rolling Max Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling max' function: ops.Max - docstring: ' Rolling Max Index Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling max index' function: ops.IdxMax - docstring: null function: ops._load_internal - docstring: ' Rolling Min Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling min' function: ops.Min - docstring: ' Rolling Min Index Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling min index' function: ops.IdxMin - docstring: null function: ops._load_internal - docstring: ' Rolling Quantile Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling quantile' function: ops.Quantile - docstring: null function: ops._load_internal - docstring: ' Rolling Median Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling median' function: ops.Med - docstring: ' Rolling Mean Absolute Deviation Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling mean absolute deviation' function: ops.Mad - docstring: null function: ops._load_internal - docstring: null function: ops.mad - docstring: ' Rolling Rank (Percentile) Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling rank' function: ops.Rank - docstring: null function: ops._load_internal - docstring: null function: ops.rank - docstring: ' Rolling Count Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling count of number of non-NaN elements' function: ops.Count - docstring: ' Rolling Delta Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with end minus start in rolling window' function: ops.Delta - docstring: null function: ops._load_internal - docstring: ' Rolling Slope This operator calculate the slope between `idx` and `feature`.(e.g. [, , ] and [1, 2, 3])Usage Example:- "Slope($close, %d)/$close"# TODO:# Some users may want pair-wise rolling like `Slope(A, B, N)`Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with linear regression slope of given window' function: ops.Slope - docstring: null function: ops._load_internal - docstring: ' Rolling R-value Square Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with linear regression r-value square of given window' function: ops.Rsquare - docstring: null function: ops._load_internal - docstring: ' Rolling Regression Residuals Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with regression residuals of given window' function: ops.Resi - docstring: null function: ops._load_internal - docstring: ' Rolling WMA Parameters----------feature : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with weighted moving average output' function: ops.WMA - docstring: null function: ops._load_internal - docstring: null function: ops.weighted_mean - docstring: ' Rolling Exponential Mean (EMA) Parameters----------feature : Expressionfeature instanceN : int, floatrolling window sizeReturns----------Expressiona feature instance with regression r-value square of given window' function: ops.EMA - docstring: null function: ops._load_internal - docstring: null function: ops.exp_weighted_mean - docstring: ' Pair Rolling Operator Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling output of two input features' function: ops.PairRolling - docstring: null function: ops._load_internal - docstring: null function: ops.get_longest_back_rolling - docstring: null function: ops.get_extended_window_size - docstring: ' Rolling Correlation Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling correlation of two input features' function: ops.Corr - docstring: null function: ops._load_internal - docstring: ' Rolling Covariance Parameters----------feature_left : Expressionfeature instancefeature_right : Expressionfeature instanceN : introlling window sizeReturns----------Expressiona feature instance with rolling max of two input features' function: ops.Cov - docstring: " \nResampling the data to target frequency.The resample function\ \ of pandas is used.- the timestamp will be at the start of the time span after\ \ resample.Parameters----------feature : ExpressionAn expression for calculating\ \ the featurefreq : strIt will be passed into the resample method for resampling\ \ basedn on given frequencyfunc : methodThe method to get the resampled valuesSome\ \ expression are high frequently used" function: ops.TResample - docstring: null function: ops._load_internal - docstring: ' Ops Wrapper self._ops = {}' function: 'ops.OpsWrapper:' - docstring: null function: ops.reset - docstring: ' register operator Parameters----------ops_list : List[Union[Type[ExpressionOps], dict]]- if type(ops_list) is List[Type[ExpressionOps]], each element of ops_list represents the operator class, which should be the subclass of `ExpressionOps`.- if type(ops_list) is List[dict], each element of ops_list represents the config of operator, which has the following format:.. code-block:: text{"class": class_name,"module_path": path,}Note: `class` should be the class name of operator, `module_path` should be a python module or path of file.' function: ops.register - docstring: ' A client class Provide the connection tool functions for ClientProvider.' function: 'client.Client:' - docstring: ' Connect to server. try:self.sio.connect("ws://" + self.server_host + ":" + str(self.server_port))except socketio.exceptions.ConnectionError:self.logger.error("Cannot connect to server - check your network or server status")' function: client.connect_server - docstring: ' Disconnect from server. try:self.sio.eio.disconnect(True)except Exception as e:self.logger.error("Cannot disconnect from server : %s" % e)' function: client.disconnect - docstring: ' Send a certain request to server. Parameters----------request_type : strtype of proposed request, ''calendar''/''instrument''/''feature''.request_content : dictrecords the information of the request.msg_proc_func : functhe function to process the message when receiving response, should have arg `*args`.msg_queue: QueueThe queue to pass the message after callback.' function: client.send_request - docstring: ' callback_wrapper :param *args: args[0] is the response content' function: client.request_callback - docstring: " \nThis helper class tries to make the provider based on storage\ \ backend more convenientIt is not necessary to inherent this class if that provider\ \ don't rely on the backend storage" function: 'data.ProviderBackendMixin:' - docstring: null function: data.get_default_backend - docstring: null function: data.backend_obj - docstring: ' Calendar provider base class Provide calendar data.' function: data.CalendarProvider - docstring: ' Get calendar of certain market in given time range. Parameters----------start_time : strstart of the time range.end_time : strend of the time range.freq : strtime frequency, available: year/quarter/month/week/day.future : boolwhether including future trading day.Returns----------listcalendar list' function: data.calendar - docstring: ' Locate the start time index and end time index in a calendar under certain frequency. Parameters----------start_time : pd.Timestampstart of the time range.end_time : pd.Timestampend of the time range.freq : strtime frequency, available: year/quarter/month/week/day.future : boolwhether including future trading day.Returns-------pd.Timestampthe real start time.pd.Timestampthe real end time.intthe index of start time.intthe index of end time.' function: data.locate_index - docstring: ' Load calendar using memcache. Parameters----------freq : strfrequency of read calendar file.future : boolwhether including future trading day.Returns-------listlist of timestamps.dictdict composed by timestamp as key and index as value for fast search.' function: data._get_calendar - docstring: ' Get the uri of calendar generation task. return hash_args(start_time, end_time, freq, future)' function: data._uri - docstring: ' Load original calendar timestamp from file. Parameters----------freq : strfrequency of read calendar file.future: boolReturns----------listlist of timestamps' function: data.load_calendar - docstring: ' Instrument provider base class Provide instrument data.' function: data.InstrumentProvider - docstring: ' Get the general config dictionary for a base market adding several dynamic filters. Parameters----------market : Union[List, str]str:market/industry/index shortname, e.g. all/sse/szse/sse50/csi300/csi500.list:["ID1", "ID2"]. A list of stocksfilter_pipe : listthe list of dynamic filters.Returns----------dict: if isinstance(market, str)dict of stockpool config.{`market` => base market name, `filter_pipe` => list of filters}example :.. code-block::{''market'': ''csi500'',''filter_pipe'': [{''filter_type'': ''ExpressionDFilter'',''rule_expression'': ''$open<40'',''filter_start_time'': None,''filter_end_time'': None,''keep'': False},{''filter_type'': ''NameDFilter'',''name_rule_re'': ''SH[0-9]{4}55'',''filter_start_time'': None,''filter_end_time'': None}]}list: if isinstance(market, list)just return the original list directly.NOTE: this will make the instruments compatible with more cases. The user code will be simpler.' function: data.instruments - docstring: ' List the instruments based on a certain stockpool config. Parameters----------instruments : dictstockpool config.start_time : strstart of the time range.end_time : strend of the time range.as_list : boolreturn instruments as list or dict.Returns-------dict or listinstruments list or dictionary with time spans' function: data.list_instruments - docstring: null function: data._uri - docstring: null function: data.get_inst_type - docstring: ' Feature provider class Provide feature data.' function: data.FeatureProvider - docstring: ' Get feature data. Parameters----------instrument : stra certain instrument.field : stra certain field of feature.start_time : strstart of the time range.end_time : strend of the time range.freq : strtime frequency, available: year/quarter/month/week/day.Returns-------pd.Seriesdata of a certain feature' function: data.feature - docstring: null function: data.PITProvider - docstring: " \nget the historical periods data series between `start_index`\ \ and `end_index`Parameters----------start_index: intstart_index is a relative\ \ index to the latest period to cur_timeend_index: intend_index is a relative\ \ index to the latest period to cur_timein most cases, the start_index and end_index\ \ will be a non-positive valuesFor example, start_index == -3 end_index == 0 and\ \ current period index is cur_idx,then the data between [start_index + cur_idx,\ \ end_index + cur_idx] will be retrieved.period: intThis is used for query specific\ \ period.The period is represented with int in Qlib. (e.g. 202001 may represent\ \ the first quarter in 2020)NOTE: `period` will override `start_index` and `end_index`Returns-------pd.SeriesThe\ \ index will be integers to indicate the periods of the dataAn typical examples\ \ will beTODORaises------FileNotFoundErrorThis exception will be raised if the\ \ queried data do not exist." function: data.period_feature - docstring: ' Expression provider class Provide Expression data.' function: data.ExpressionProvider - docstring: null function: data.get_expression_instance - docstring: ' Get Expression data. The responsibility of `expression`- parse the `field` and `load` the according data.- When loading the data, it should handle the time dependency of the data. `get_expression_instance` is commonly used in this methodParameters----------instrument : stra certain instrument.field : stra certain field of feature.start_time : strstart of the time range.end_time : strend of the time range.freq : strtime frequency, available: year/quarter/month/week/day.Returns-------pd.Seriesdata of a certain expressionThe data has two types of format1) expression with datetime index2) expression with integer index- because the datetime is not as good as' function: data.expression - docstring: ' Dataset provider class Provide Dataset data.' function: data.DatasetProvider - docstring: ' Get dataset data. Parameters----------instruments : list or dictlist/dict of instruments or dict of stockpool config.fields : listlist of feature instances.start_time : strstart of the time range.end_time : strend of the time range.freq : strtime frequency.inst_processors: Iterable[Union[dict, InstProcessor]]the operations performed on each instrumentReturns----------pd.DataFramea pandas dataframe with index.' function: data.dataset - docstring: ' Get task uri, used when generating rabbitmq task in qlib_server Parameters----------instruments : list or dictlist/dict of instruments or dict of stockpool config.fields : listlist of feature instances.start_time : strstart of the time range.end_time : strend of the time range.freq : strtime frequency.disk_cache : intwhether to skip(0)/use(1)/replace(2) disk_cache.' function: data._uri - docstring: " \nParse different types of input instruments to output instruments_dWrong\ \ format of input instruments will lead to exception." function: data.get_instruments_d - docstring: " \nGet column names from input fields" function: data.get_column_names - docstring: null function: data.parse_fields - docstring: " \nLoad and process the data, return the data set.- default using\ \ multi-kernel method." function: data.dataset_processor - docstring: " \nCalculate the expressions for **one** instrument, return a\ \ df result.If the expression has been calculated before, load from cache.return\ \ value: A data frame with index 'datetime' and other data columns." function: data.inst_calculator - docstring: ' Local calendar data provider class Provide calendar data from local data source.' function: data.LocalCalendarProvider - docstring: ' Load original calendar timestamp from file. Parameters----------freq : strfrequency of read calendar file.future: boolReturns----------listlist of timestamps' function: data.load_calendar - docstring: ' Local instrument data provider class Provide instrument data from local data source.' function: data.LocalInstrumentProvider - docstring: null function: data._load_instruments - docstring: null function: data.list_instruments - docstring: ' Local feature data provider class Provide feature data from local data source.' function: data.LocalFeatureProvider - docstring: null function: data.feature - docstring: null function: data.LocalPITProvider - docstring: null function: data.period_feature - docstring: ' Local expression data provider class Provide expression data from local data source.' function: data.LocalExpressionProvider - docstring: null function: data.expression - docstring: ' Local dataset data provider class Provide dataset data from local data source.' function: data.LocalDatasetProvider - docstring: null function: data.dataset - docstring: " \nThis method is used to prepare the expression cache for the\ \ client.Then the client will load the data from expression cache by itself." function: data.multi_cache_walker - docstring: " \nIf the expressions of one instrument haven't been calculated\ \ before,calculate it and write it into expression cache." function: data.cache_walker - docstring: ' Client calendar data provider class Provide calendar data by requesting data from server as a client.' function: data.ClientCalendarProvider - docstring: null function: data.set_conn - docstring: null function: data.calendar - docstring: ' Client instrument data provider class Provide instrument data by requesting data from server as a client.' function: data.ClientInstrumentProvider - docstring: null function: data.set_conn - docstring: null function: data.list_instruments - docstring: null function: data.inst_msg_proc_func - docstring: ' Client dataset data provider class Provide dataset data by requesting data from server as a client.' function: data.ClientDatasetProvider - docstring: null function: data.set_conn - docstring: " \nCall the server to generate the expression cache.Then\ \ load the data from the expression cache directly.- default using multi-kernel\ \ method." function: data.dataset - docstring: ' Local provider class It is a set of interface that allow users to access data.Because PITD is not exposed publicly to users, so it is not included in the interface.To keep compatible with old qlib provider.' function: 'data.BaseProvider:' - docstring: null function: data.calendar - docstring: null function: data.instruments - docstring: null function: data.list_instruments - docstring: " \nParameters----------disk_cache : intwhether to skip(0)/use(1)/replace(2)\ \ disk_cacheThis function will try to use cache method which has a keyword `disk_cache`,and\ \ will use provider method if a type error is raised because the DatasetD instanceis\ \ a provider class." function: data.features - docstring: null function: data.LocalProvider - docstring: ' _uri The server hope to get the uri of the request. The uri will be decidedby the dataprovider. For ex, different cache layer has different uri.:param type: The type of resource for the uri:param **kwargs:' function: data._uri - docstring: ' features_uri Return the uri of the generated cache of features/dataset:param disk_cache::param instruments::param fields::param start_time::param end_time::param freq:' function: data.features_uri - docstring: ' Client Provider Requesting data from server as a client. Can propose requests:- Calendar : Directly respond a list of calendars- Instruments (without filter): Directly respond a list/dict of instruments- Instruments (with filters): Respond a list/dict of instruments- Features : Respond a cache uriThe general workflow is described as follows:When the user use client provider to propose a request, the client provider will connect the server and send the request. The client will start to wait for the response. The response will be made instantly indicating whether the cache is available. The waiting procedure will terminate only when the client get the response saying `feature_available` is true.`BUG` : Everytime we make request for certain data we need to connect to the server, wait for the response and disconnect from it. We can''t make a sequence of requests within one connection. You can refer to https://python-socketio.readthedocs.io/en/latest/client.html for documentation of python-socketIO client.' function: data.ClientProvider - docstring: null function: data.is_instance_of_provider - docstring: " \nExpression base classExpression is designed to handle the calculation\ \ of data with the format belowdata with two dimension for each instrument,- feature-\ \ time: it could be observation time or period time.- period time is designed\ \ for Point-in-time database. For example, the period time maybe 2014Q4, its\ \ value can observed for multiple times(different value may be observed at different\ \ time due to amendment)." function: base.Expression - docstring: ' load feature This function is responsible for loading feature/expression based on the expression engine.The concrete implementation will be separated into two parts:1) caching data, handle errors.- This part is shared by all the expressions and implemented in Expression2) processing and calculating data based on the specific expression.- This part is different in each expression and implemented in each expressionExpression Engine is shared by different data.Different data will have different extra information for `args`.Parameters----------instrument : strinstrument code.start_index : strfeature start index [in calendar].end_index : strfeature end index [in calendar].*args may contain following information:1) if it is used in basic expression engine data, it contains following argumentsfreq: strfeature frequency.2) if is used in PIT data, it contains following argumentscur_pit:it is designed for the point-in-time data.period: intThis is used for query specific period.The period is represented with int in Qlib. (e.g. 202001 may represent the first quarter in 2020)Returns----------pd.Seriesfeature series: The index of the series is the calendar index' function: base.load - docstring: null function: base._load_internal - docstring: ' Get the longest length of historical data the feature has accessed This is designed for getting the needed range of the data to calculatethe features in specific range at first. However, situations likeRef(Ref($close, -1), 1) can not be handled rightly.So this will only used for detecting the length of historical data needed.' function: base.get_longest_back_rolling - docstring: ' get_extend_window_size For to calculate this Operator in range[start_index, end_index]We have to get the *leaf feature* inrange[start_index - lft_etd, end_index + rght_etd].Returns----------(int, int)lft_etd, rght_etd' function: base.get_extended_window_size - docstring: ' Static Expression This kind of feature will load data from provider' function: base.Feature - docstring: null function: base._load_internal - docstring: null function: base.get_longest_back_rolling - docstring: null function: base.get_extended_window_size - docstring: null function: base.PFeature - docstring: null function: base._load_internal - docstring: ' Operator Expression This kind of feature will use operator for featureconstruction on the fly.' function: base.ExpressionOps - docstring: " \nprocess the dataNOTE: **The processor could change the content\ \ of `df` inplace !!!!! **User should keep a copy of data outsideParameters----------df\ \ : pd.DataFrameThe raw_df of handler or result from previous processor." function: 'inst_processor.InstProcessor:' - docstring: ' Dynamic Instruments Filter Abstract class Users can override this class to construct their own filterOverride __init__ to input filter regulationsOverride filter_main to use the regulations to filter instruments' function: filter.BaseDFilter - docstring: ' Construct an instance from config dict. Parameters----------config : dictdict of config parameters.' function: filter.from_config - docstring: ' Construct an instance from config dict. Returns----------dictreturn the dict of config parameters.' function: filter.to_config - docstring: ' Dynamic Instruments Filter Abstract class to filter a series of certain features Filters should provide parameters:- filter start time- filter end time- filter ruleOverride __init__ to assign a certain rule to filter the series.Override _getFilterSeries to use the rule to filter the series and get a dict of {inst => series}, or override filter_main for more advanced series filter rule' function: filter.SeriesDFilter - docstring: ' Get time bound for all instruments. Parameters----------instruments: dictthe dict of instruments in the form {instrument_name => list of timestamp tuple}.Returns----------pd.Timestamp, pd.Timestampthe lower time bound and upper time bound of all the instruments.' function: filter._getTimeBound - docstring: ' Convert the target timestamp to a pandas series of bool value within a time range. Make the time inside the target_timestamp range TRUE, others FALSE.Parameters----------time_range : D.calendarthe time range of the instruments.target_timestamp : listthe list of tuple (timestamp, timestamp).Returns----------pd.Seriesthe series of bool value for an instrument.' function: filter._toSeries - docstring: ' Filter the timestamp series with filter series by using element-wise AND operation of the two series. Parameters----------timestamp_series : pd.Seriesthe series of bool value indicating existing time.filter_series : pd.Seriesthe series of bool value indicating filter feature.Returns----------pd.Seriesthe series of bool value indicating whether the date satisfies the filter condition and exists in target timestamp.' function: filter._filterSeries - docstring: ' Convert the timestamp series to a list of tuple (timestamp, timestamp) indicating a continuous range of TRUE. Parameters----------timestamp_series: pd.Seriesthe series of bool value after being filtered.Returns----------listthe list of tuple (timestamp, timestamp).' function: filter._toTimestamp - docstring: ' Get filter series based on the rules assigned during the initialization and the input time range. Parameters----------instruments : dictthe dict of instruments to be filtered.fstart : pd.Timestampstart time of filter.fend : pd.Timestampend time of filter... note:: fstart/fend indicates the intersection of instruments start/end time and filter start/end time.Returns----------pd.Dataframea series of {pd.Timestamp => bool}.' function: filter._getFilterSeries - docstring: ' Implement this method to filter the instruments. Parameters----------instruments: dictinput instruments to be filtered.start_time: strstart of the time range.end_time: strend of the time range.Returns----------dictfiltered instruments, same structure as input instruments.' function: filter.filter_main - docstring: ' Name dynamic instrument filter Filter the instruments based on a regulated name format.A name rule regular expression is required.' function: filter.NameDFilter - docstring: null function: filter._getFilterSeries - docstring: null function: filter.from_config - docstring: null function: filter.to_config - docstring: ' Expression dynamic instrument filter Filter the instruments based on a certain expression.An expression rule indicating a certain feature field is required.Examples----------- *basic features filter* : rule_expression = ''$close/$open>5''- *cross-sectional features filter* : rule_expression = ''$rank($close)<10''- *time-sequence features filter* : rule_expression = ''$Ref($close, 3)>100''' function: filter.ExpressionDFilter - docstring: null function: filter._getFilterSeries - docstring: null function: filter.from_config - docstring: null function: pit.P - docstring: null function: pit._load_internal - docstring: null function: pit._load_feature - docstring: null function: pit.get_longest_back_rolling - docstring: null function: pit.get_extended_window_size - docstring: null function: pit.PRef - docstring: null function: cache.QlibCacheException - docstring: ' Memory Cache Unit. self.size_limit = kwargs.pop("size_limit", 0)self._size = 0self.od = OrderedDict()# TODO: thread safe?__setitem__ failure might cause inconsistent size?# precalculate the size after od.__setitem__self._adjust_size(key, value)self.od.__setitem__(key, value)# move the key to end,make it latestself.od.move_to_end(key)if self.limited:# pop the oldest items beyond size limitwhile self._size > self.size_limit:self.popitem(last=False)v = self.od.__getitem__(key)self.od.move_to_end(key)return vreturn key in self.odreturn self.od.__len__()return f"{self.__class__.__name__}\n{self.od.__repr__()}"' function: cache.MemCacheUnit - docstring: null function: cache.set_limit_size - docstring: ' whether memory cache is limited return self.size_limit > 0@property' function: cache.limited - docstring: null function: cache.total_size - docstring: null function: cache.clear - docstring: null function: cache.popitem - docstring: null function: cache.pop - docstring: null function: cache._adjust_size - docstring: null function: cache._get_value_size - docstring: null function: cache.MemCacheLengthUnit - docstring: null function: cache._get_value_size - docstring: null function: cache.MemCacheSizeofUnit - docstring: null function: cache._get_value_size - docstring: ' Memory cache. ' function: 'cache.MemCache:' - docstring: null function: cache.clear - docstring: null function: 'cache.MemCacheExpire:' - docstring: ' set cache :param mem_cache: MemCache attribute(''c''/''i''/''f'').:param key: cache key.:param value: cache value.' function: cache.set_cache - docstring: ' get mem cache :param mem_cache: MemCache attribute(''c''/''i''/''f'').:param key: cache key.:return: cache value; if cache not exist, return None.' function: cache.get_cache - docstring: null function: 'cache.CacheUtils:' - docstring: null function: cache.organize_meta_file - docstring: null function: cache.reset_lock - docstring: null function: cache.visit - docstring: " \n) from lock_acquired@staticmethod@contextlib.contextmanager" function: cache.acquire - docstring: null function: cache.reader_lock - docstring: null function: cache.writer_lock - docstring: ' Provider cache base class self.provider = providerself.logger = get_module_logger(self.__class__.__name__)return getattr(self.provider, attr)@staticmethod' function: 'cache.BaseProviderCache:' - docstring: null function: cache.check_cache_exists - docstring: null function: cache.clear_cache - docstring: null function: cache.get_cache_dir - docstring: ' Expression cache mechanism base class. This class is used to wrap expression provider with self-defined expression cache mechanism... note:: Override the `_uri` and `_expression` method to create your own expression cache mechanism.' function: cache.ExpressionCache - docstring: ' Get expression data. .. note:: Same interface as `expression` method in expression provider' function: cache.expression - docstring: ' Get expression cache file uri. Override this method to define how to get expression cache file uri corresponding to users'' own cache mechanism.' function: cache._uri - docstring: ' Get expression data using cache. Override this method to define how to get expression data corresponding to users'' own cache mechanism.' function: cache._expression - docstring: ' Update expression cache to latest calendar. Override this method to define how to update expression cache corresponding to users'' own cache mechanism.Parameters----------cache_uri : str or Paththe complete uri of expression cache file (include dir path).freq : strReturns-------int0(successful update)/ 1(no need to update)/ 2(update failure).' function: cache.update - docstring: ' Dataset cache mechanism base class. This class is used to wrap dataset provider with self-defined dataset cache mechanism... note:: Override the `_uri` and `_dataset` method to create your own dataset cache mechanism.' function: cache.DatasetCache - docstring: ' Get feature dataset. .. note:: Same interface as `dataset` method in dataset provider.. note:: The server use redis_lock to make sureread-write conflicts will not be triggeredbut client readers are not considered.' function: cache.dataset - docstring: ' Get dataset cache file uri. Override this method to define how to get dataset cache file uri corresponding to users'' own cache mechanism.' function: cache._uri - docstring: ' Get feature dataset using cache. Override this method to define how to get feature dataset corresponding to users'' own cache mechanism.' function: cache._dataset - docstring: ' Get a uri of feature dataset using cache. specially:disk_cache=1 means using data set cache and return the uri of cache file.disk_cache=0 means client knows the path of expression cache,server checks if the cache exists(if not, generate it), and client loads data by itself.Override this method to define how to get feature dataset uri corresponding to users'' own cache mechanism.' function: cache._dataset_uri - docstring: ' Update dataset cache to latest calendar. Override this method to define how to update dataset cache corresponding to users'' own cache mechanism.Parameters----------cache_uri : str or Paththe complete uri of dataset cache file (include dir path).freq : strReturns-------int0(successful update)/ 1(no need to update)/ 2(update failure)' function: cache.update - docstring: ' cache data to origin data :param data: pd.DataFrame, cache data.:param fields: feature fields.:return: pd.DataFrame.' function: cache.cache_to_origin_data - docstring: ' normalize uri args instruments = normalize_cache_instruments(instruments)fields = normalize_cache_fields(fields)freq = freq.lower()return instruments, fields, freq' function: cache.normalize_uri_args - docstring: ' Prepared cache mechanism for server. super(DiskExpressionCache, self).__init__(provider)self.r = get_redis_connection()# remote==True means client is using this module, writing behaviour will not be allowed.self.remote = kwargs.get("remote", False)' function: cache.DiskExpressionCache - docstring: null function: cache.get_cache_dir - docstring: null function: cache._uri - docstring: " \nIn most cases, we do not need reader_lock.Because updating\ \ data is a small probability event compare to reading data." function: cache._expression - docstring: ' use bin file to save like feature-data. # Make sure the cache runs right when the directory is deleted# while runningmeta = {"info": {"instrument": instrument, "field": field, "freq": freq, "last_update": last_update},"meta": {"last_visit": time.time(), "visits": 1},}self.logger.debug(f"generating expression cache: {meta}")self.clear_cache(cache_path)meta_path = cache_path.with_suffix(".meta")with meta_path.open("wb") as f:pickle.dump(meta, f, protocol=C.dump_protocol_version)meta_path.chmod(stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH)df = expression_data.to_frame()r = np.hstack([df.index[0], expression_data]).astype(" with a timestamp as its index.- It indicates the `start_index` (included) and `end_index` (excluded) of the data for `timestamp`- meta data: cache/d41366901e25de3ec47297f12e2ba11d.meta- data : cache/d41366901e25de3ec47297f12e2ba11d- This is a hdf file sorted by datetime:param cache_path: The path to store the cache.:param instruments: The instruments to store the cache.:param fields: The fields to store the cache.:param freq: The freq to store the cache.:param inst_processors: Instrument processors.:return type pd.DataFrame; The fields of the returned DataFrame are consistent with the parameters of the function.' function: cache.gen_dataset_cache - docstring: null function: cache.update - docstring: ' Simple dataset cache that can be used locally or on client. super(SimpleDatasetCache, self).__init__(provider)try:self.local_cache_path: Path = Path(C["local_cache_path"]).expanduser().resolve()except (KeyError, TypeError):self.logger.error("Assign a local_cache_path in config if you want to use this cache mechanism")raiseself.logger.info(f"DatasetCache directory: {self.local_cache_path}, "f"modify the cache directory via the local_cache_path in the config")' function: cache.SimpleDatasetCache - docstring: null function: cache._uri - docstring: null function: cache._dataset - docstring: ' Prepared cache mechanism for server. ' function: cache.DatasetURICache - docstring: null function: cache._uri - docstring: null function: cache.dataset - docstring: null function: cache.CalendarCache - docstring: null function: cache.MemoryCalendarCache - docstring: ' FileStorageMixin, applicable to FileXXXStorage Subclasses need to have provider_uri, freq, storage_name, file_name attributes' function: 'file_storage.FileStorageMixin:' - docstring: null function: file_storage.provider_uri - docstring: null function: file_storage.dpm - docstring: null function: file_storage.support_freq - docstring: null function: file_storage.uri - docstring: ' check self.uri Raises-------ValueError' function: file_storage.check - docstring: null function: file_storage.FileCalendarStorage - docstring: null function: file_storage.file_name - docstring: ' the freq to read from file if not hasattr(self, "_freq_file_cache"):freq = Freq(self.freq)if freq not in self.support_freq:# NOTE: uri# 1. If `uri` does not exist# - Get the `min_uri` of the closest `freq` under the same "directory" as the `uri`# - Read data from `min_uri` and resample to `freq`freq = Freq.get_recent_freq(freq, self.support_freq)if freq is None:raise ValueError(f"can''t find a freq from {self.support_freq} that can resample to {self.freq}!")self._freq_file_cache = freqreturn self._freq_file_cache' function: file_storage._freq_file - docstring: null function: file_storage._read_calendar - docstring: null function: file_storage._write_calendar - docstring: null function: file_storage.uri - docstring: null function: file_storage.data - docstring: null function: file_storage._get_storage_freq - docstring: null function: file_storage.extend - docstring: null function: file_storage.clear - docstring: null function: file_storage.index - docstring: null function: file_storage.insert - docstring: null function: file_storage.remove - docstring: null function: file_storage.FileInstrumentStorage - docstring: null function: file_storage._read_instrument - docstring: null function: file_storage._write_instrument - docstring: null function: file_storage.clear - docstring: null function: file_storage.data - docstring: null function: file_storage.update - docstring: null function: file_storage.FileFeatureStorage - docstring: null function: file_storage.clear - docstring: null function: file_storage.data - docstring: null function: file_storage.write - docstring: null function: file_storage.start_index - docstring: null function: storage.UserCalendarStorage - docstring: null function: storage.data - docstring: null function: storage.UserInstrumentStorage - docstring: null function: storage.data - docstring: ' ' function: storage.UserFeatureStorage - docstring: null function: 'storage.BaseStorage:' - docstring: null function: storage.storage_name - docstring: " \nThe behavior of CalendarStorage's methods and List's methods of\ \ the same name remain consistent" function: storage.CalendarStorage - docstring: ' get all data Raises------ValueErrorIf the data(storage) does not exist, raise ValueError' function: storage.data - docstring: null function: storage.clear - docstring: null function: storage.extend - docstring: " \nRaises------ValueErrorIf the data(storage) does not exist,\ \ raise ValueError" function: storage.index - docstring: null function: storage.insert - docstring: ' x.__setitem__(i, o) <==> (x[i] = o) @overloadx.__setitem__(s, o) <==> (x[s] = o)' function: storage.remove - docstring: null function: storage.InstrumentStorage - docstring: ' get all data Raises------ValueErrorIf the data(storage) does not exist, raise ValueError' function: storage.data - docstring: null function: storage.clear - docstring: ' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. Notes------If E present and has a .keys() method, does: for k in E: D[k] = E[k]If E present and lacks .keys() method, does: for (k, v) in E: D[k] = vIn either case, this is followed by: for k, v in F.items(): D[k] = v' function: storage.update - docstring: null function: storage.FeatureStorage - docstring: ' get all data Notes------if data(storage) does not exist, return empty pd.Series: `return pd.Series(dtype=np.float32)`' function: storage.data - docstring: ' get FeatureStorage start index Notes-----If the data(storage) does not exist, return None' function: storage.start_index - docstring: ' get FeatureStorage end index Notes-----The right index of the data range (both sides are closed)The next data appending point will be `end_index + 1`If the data(storage) does not exist, return None' function: storage.end_index - docstring: null function: storage.clear - docstring: ' Write data_array to FeatureStorage starting from index. Notes------If index is None, append data_array to feature.If len(data_array) == 0; returnIf (index - self.end_index) >= 1, self[end_index+1: index] will be filled with np.nanExamples---------.. code-block::feature:3 44 55 6>>> self.write([6, 7], index=6)feature:3 44 55 66 67 7>>> self.write([8], index=9)feature:3 44 55 66 67 78 np.nan9 8>>> self.write([1, np.nan], index=3)feature:3 14 np.nan5 66 67 78 np.nan9 8' function: storage.write - docstring: ' Rebase the start_index and end_index of the FeatureStorage. start_index and end_index are closed intervals: [start_index, end_index]Examples---------.. code-block::feature:3 44 55 6>>> self.rebase(start_index=4)feature:4 55 6>>> self.rebase(start_index=3)feature:3 np.nan4 55 6>>> self.write([3], index=3)feature:3 34 55 6>>> self.rebase(end_index=4)feature:3 34 5>>> self.write([6, 7, 8], index=4)feature:3 34 65 76 8>>> self.rebase(start_index=4, end_index=5)feature:4 65 7' function: storage.rebase - docstring: ' overwrite all data in FeatureStorage with data Parameters----------data: Union[List, np.ndarray, Tuple]dataindex: intdata start index' function: storage.rewrite - docstring: " \nDataLoader is designed for loading raw data from original data\ \ source." function: loader.DataLoader - docstring: " \nload the data as pd.DataFrame.Example of the data (The multi-index\ \ of the columns is optional.):.. code-block:: textfeature \ \ label$close $volume Ref($close,\ \ 1) Mean($close, 3) $high-$low LABEL0datetime instrument2010-01-04 SH600000\ \ 81.807068 17145150.0 83.737389 83.016739 2.741058 0.0032SH600004\ \ 13.313329 11800983.0 13.313329 13.317701 0.183632 0.0042SH600005\ \ 37.796539 12231662.0 38.258602 37.919757 0.970325 0.0289Parameters----------instruments\ \ : str or dictit can either be the market name or the config file of instruments\ \ generated by InstrumentProvider.start_time : strstart of the time range.end_time\ \ : strend of the time range.Returns-------pd.DataFrame:data load from the under\ \ layer source" function: loader.load - docstring: " \n(D)ata(L)oader (W)ith (P)arser for features and namesExtracting\ \ this class so that QlibDataLoader and other dataloaders(such as QdbDataLoader)\ \ can share the fields." function: loader.DLWParser - docstring: null function: loader._parse_fields_info - docstring: " \nload the dataframe for specific groupParameters----------instruments\ \ :the instruments.exprs : listthe expressions to describe the content of the\ \ data.names : listthe name of the data.Returns-------pd.DataFrame:the queried\ \ dataframe." function: loader.load_group_df - docstring: null function: loader.load - docstring: ' Same as QlibDataLoader. The fields can be define by config self,config: Tuple[list, tuple, dict],filter_pipe: List = None,swap_level: bool = True,freq: Union[str, dict] = "day",inst_processors: Union[dict, list] = None,):' function: loader.QlibDataLoader - docstring: null function: loader.load_group_df - docstring: " \nDataLoader that supports loading data from file or as provided." function: loader.StaticDataLoader - docstring: null function: loader.load - docstring: null function: loader._maybe_load_raw_data - docstring: ' DataLoaderDH DataLoader based on (D)ata (H)andlerIt is designed to load multiple data from data handler- If you just want to load data from single datahandler, you can write them in single data handlerTODO: What make this module not that easy to use.- For online scenario- The underlayer data handler should be configured. But data loader doesn''t provide such interface & hook.' function: loader.DataLoaderDH - docstring: " \nThe steps to using a handler1. initialized data handler (call\ \ by `init`).2. use the data.The data handler try to maintain a handler with 2\ \ level.`datetime` & `instruments`.Any order of the index level can be supported\ \ (The order will be implied in the data).The order <`datetime`, `instruments`>\ \ will be used when the dataframe index name is missed.Example of the data:The\ \ multi-index of the columns is optional... code-block:: textfeature \ \ label$close $volume Ref($close,\ \ 1) Mean($close, 3) $high-$low LABEL0datetime instrument2010-01-04 SH600000\ \ 81.807068 17145150.0 83.737389 83.016739 2.741058 0.0032SH600004\ \ 13.313329 11800983.0 13.313329 13.317701 0.183632 0.0042SH600005\ \ 37.796539 12231662.0 38.258602 37.919757 0.970325 0.0289Tips\ \ for improving the performance of datahandler- Fetching data with `col_set=CS_RAW`\ \ will return the raw data and may avoid pandas from copying the data when calling\ \ `loc`" function: handler.DataHandler - docstring: " \nconfiguration of data.# what data to be loaded from data sourceThis\ \ method will be used when loading pickled handler from dataset.The data will\ \ be initialized with different time range." function: handler.config - docstring: " \nSet Up the data in case of running initialization for multiple\ \ timeIt is responsible for maintaining following variable1) self._dataParameters----------enable_cache\ \ : booldefault value is false:- if `enable_cache` == True:the processed data\ \ will be saved on disk, and handler will load the cached data from the disk directlywhen\ \ we call `init` next time" function: handler.setup_data - docstring: " \nfetch data from underlying data sourceDesign motivation:-\ \ providing a unified interface for underlying data.- Potential to make the interface\ \ more friendly.- User can improve performance when fetching data in this extra\ \ layerParameters----------selector : Union[pd.Timestamp, slice, str]describe\ \ how to select data by indexIt can be categories as following- fetch single index-\ \ fetch a range of index- a slice range- pd.Index for specific indexesFollowing\ \ conflicts may occur- Does [\"20200101\", \"20210101\"] mean selecting this slice\ \ or these two days?- slice have higher prioritieslevel : Union[str, int]which\ \ index level to select the datacol_set : Union[str, List[str]]- if isinstance(col_set,\ \ str):select a set of meaningful, pd.Index columns.(e.g. features, columns)-\ \ if col_set == CS_RAW:the raw dataset will be returned.- if isinstance(col_set,\ \ List[str]):select several sets of meaningful columns, the returned data has\ \ multiple levelsproc_func: Callable- Give a hook for processing data before fetching-\ \ An example to explain the necessity of the hook:- A Dataset learned some processors\ \ to process data which is related to data segmentation- It will apply them every\ \ time when preparing data.- The learned processor require the dataframe remains\ \ the same format when fitting and applying- However the data format will change\ \ according to the parameters.- So the processors should be applied to the underlayer\ \ data.squeeze : boolwhether squeeze columns and indexReturns-------pd.DataFrame." function: handler.fetch - docstring: null function: handler._fetch_data - docstring: " \nget the column namesParameters----------col_set : strselect\ \ a set of meaningful columns.(e.g. features, columns)Returns-------list:list\ \ of column names" function: handler.get_cols - docstring: " \nget range selector by number of periodsArgs:cur_date (pd.Timestamp\ \ or str): current dateperiods (int): number of periods" function: handler.get_range_selector - docstring: " \nget an iterator of sliced data with given periodsArgs:periods\ \ (int): number of periods.min_periods (int): minimum periods for sliced dataframe.kwargs\ \ (dict): will be passed to `self.fetch`." function: handler.get_range_iterator - docstring: " \nDataHandler with **(L)earnable (P)rocessor**This handler will\ \ produce three pieces of data in pd.DataFrame format.- DK_R / self._data: the\ \ raw data loaded from the loader- DK_I / self._infer: the data processed for\ \ inference- DK_L / self._learn: the data processed for learning model.The motivation\ \ of using different processor workflows for learning and inferenceHere are some\ \ examples.- The instrument universe for learning and inference may be different.-\ \ The processing of some samples may rely on label (for example, some samples\ \ hit the limit may need extra processing or be dropped).- These processors only\ \ apply to the learning phase.Tips for data handler- To reduce the memory cost-\ \ `drop_raw=True`: this will modify the data inplace on raw data;- Please note\ \ processed data like `self._infer` or `self._learn` are concepts different from\ \ `segments` in Qlib's `Dataset` like \"train\" and \"test\"- Processed data like\ \ `self._infer` or `self._learn` are underlying data processed with different\ \ processors- `segments` in Qlib's `Dataset` like \"train\" and \"test\" are simply\ \ the time segmentations when querying data(\"train\" are often before \"test\"\ \ in time-series).- For example, you can query `data._infer` processed by `infer_processors`\ \ in the \"train\" time segmentation." function: handler.DataHandlerLP - docstring: null function: handler.get_all_processors - docstring: " \nfit data without processing the data" function: handler.fit - docstring: " \nfit and process dataThe input of the `fit` will be the output\ \ of the previous processor" function: handler.fit_process_data - docstring: null function: handler._run_proc_l - docstring: " \nNOTE: it will return True if `len(proc_l) == 0`" function: handler._is_proc_readonly - docstring: " \nprocess_data data. Fun `processor.fit` if necessaryNotation:\ \ (data) [processor]# data processing flow of self.process_type == DataHandlerLP.PTYPE_I..\ \ code-block:: text(self._data)-[shared_processors]-(_shared_df)-[learn_processors]-(_learn_df)\\\ \\-[infer_processors]-(_infer_df)# data processing flow of self.process_type ==\ \ DataHandlerLP.PTYPE_A.. code-block:: text(self._data)-[shared_processors]-(_shared_df)-[infer_processors]-(_infer_df)-[learn_processors]-(_learn_df)Parameters----------with_fit\ \ : boolThe input of the `fit` will be the output of the previous processor" function: handler.process_data - docstring: " \nconfiguration of data.# what data to be loaded from data sourceThis\ \ method will be used when loading pickled handler from dataset.The data will\ \ be initialized with different time range." function: handler.config - docstring: " \nSet up the data in case of running initialization for multiple\ \ timeParameters----------init_type : strThe type `IT_*` listed above.enable_cache\ \ : booldefault value is false:- if `enable_cache` == True:the processed data\ \ will be saved on disk, and handler will load the cached data from the disk directlywhen\ \ we call `init` next time" function: handler.setup_data - docstring: null function: handler._get_df_by_key - docstring: " \nfetch data from underlying data sourceParameters----------selector\ \ : Union[pd.Timestamp, slice, str]describe how to select data by index.level\ \ : Union[str, int]which index level to select the data.col_set : strselect a\ \ set of meaningful columns.(e.g. features, columns).data_key : strthe data to\ \ fetch: DK_*.proc_func: Callableplease refer to the doc of DataHandler.fetchReturns-------pd.DataFrame:" function: handler.fetch - docstring: " \nget the column namesParameters----------col_set : strselect\ \ a set of meaningful columns.(e.g. features, columns).data_key : DATA_KEY_TYPEthe\ \ data to fetch: DK_*.Returns-------list:list of column names" function: handler.get_cols - docstring: " \nMotivation- A user creates a datahandler in his customized\ \ package. Then he wants to share the processed handler toother users without\ \ introduce the package dependency and complicated data processing logic.- This\ \ class make it possible by casting the class to DataHandlerLP and only keep the\ \ processed dataParameters----------handler : DataHandlerLPA subclass of DataHandlerLPReturns-------DataHandlerLP:the\ \ converted processed data" function: handler.cast - docstring: " \nMotivation:- When user want to get a quick data handler.The\ \ created data handler will have only one shared Dataframe without processors.After\ \ creating the handler, user may often want to dump the handler for reuseHere\ \ is a typical use case.. code-block:: pythonfrom qlib.data.dataset import DataHandlerLPdh\ \ = DataHandlerLP.from_df(df)dh.to_pickle(fname, dump_all=True)TODO:- The StaticDataLoader\ \ is quite slow. It don't have to copy the data again..." function: handler.from_df - docstring: " \nget the level index of `df` given `level`Parameters----------df\ \ : pd.DataFramedatalevel : Union[str, int]index levelReturns-------int:The level\ \ index in the multiple index" function: utils.get_level_index - docstring: " \nfetch data from `data` with `selector` and `level`selector are\ \ assumed to be well processed.`fetch_df_by_index` is only responsible for get\ \ the right levelParameters----------selector : Union[pd.Timestamp, slice, str,\ \ list]selectorlevel : Union[int, str]the level to use the selectorReturns-------Data\ \ of the given index." function: utils.fetch_df_by_index - docstring: null function: utils.fetch_df_by_col - docstring: " \nConvert the format of df.MultiIndex according to the following\ \ rules:- If `level` is the first level of df.MultiIndex, do nothing- If `level`\ \ is the second level of df.MultiIndex, swap the level of index.NOTE:the number\ \ of levels of df.MultiIndex should be 2Parameters----------df : Union[pd.DataFrame,\ \ pd.Series]raw DataFrame/Serieslevel : str, optionalthe level that will be converted\ \ to the first one, by default \"datetime\"Returns-------Union[pd.DataFrame, pd.Series]converted\ \ DataFrame/Series" function: utils.convert_index_format - docstring: " \ninitialize the handler part of the task **inplace**Parameters----------task\ \ : dictthe task to be handledReturns-------Union[DataHandler, None]:returns" function: utils.init_task_handler - docstring: " \nget a group of columns from multi-index columns DataFrameParameters----------df\ \ : pd.DataFramewith multi of columns.group : strthe name of the feature group,\ \ i.e. the first level value of the group index." function: processor.get_group_columns - docstring: null function: processor.Processor - docstring: " \nlearn data processing parametersParameters----------df : pd.DataFrameWhen\ \ we fit and process data with processor one by one. The fit function reiles on\ \ the output of previousprocessor, i.e. `df`." function: processor.fit - docstring: " \nIs this processor usable for inferenceSome processors are\ \ not usable for inference.Returns-------bool:if it is usable for infenrece." function: processor.is_for_infer - docstring: " \nDoes the processor treat the input data readonly (i.e. does\ \ not write the input data) when processingKnowning the readonly information is\ \ helpful to the Handler to avoid uncessary copy" function: processor.readonly - docstring: null function: processor.config - docstring: null function: processor.DropnaProcessor - docstring: null function: processor.readonly - docstring: null function: processor.DropnaLabel - docstring: ' The samples are dropped according to label. So it is not usable for inference return False' function: processor.is_for_infer - docstring: null function: processor.DropCol - docstring: null function: processor.readonly - docstring: null function: processor.FilterCol - docstring: null function: processor.readonly - docstring: ' Use tanh to process noise data ' function: processor.TanhProcess - docstring: null function: processor.tanh_denoise - docstring: ' Process infinity ' function: processor.ProcessInf - docstring: null function: processor.replace_inf - docstring: null function: processor.process_inf - docstring: ' Process NaN self.fields_group = fields_groupself.fill_value = fill_valueif self.fields_group is None:df.fillna(self.fill_value, inplace=True)else:cols = get_group_columns(df, self.fields_group)# this implementation is extremely slow# df.fillna({col: self.fill_value for col in cols}, inplace=True)# So we use numpy to accelerate filling valuesnan_select = np.isnan(df.values)nan_select[:, ~df.columns.isin(cols)] = Falsedf.values[nan_select] = self.fill_valuereturn df' function: processor.Fillna - docstring: null function: processor.MinMaxNorm - docstring: null function: processor.fit - docstring: null function: processor.normalize - docstring: ' ZScore Normalization # NOTE: correctly set the `fit_start_time` and `fit_end_time` is very important !!!# `fit_end_time` **must not** include any information from the test data!!!self.fit_start_time = fit_start_timeself.fit_end_time = fit_end_timeself.fields_group = fields_group' function: processor.ZScoreNorm - docstring: null function: processor.fit - docstring: null function: processor.normalize - docstring: ' Robust ZScore Normalization Use robust statistics for Z-Score normalization:mean(x) = median(x)std(x) = MAD(x) * 1.4826Reference:https://en.wikipedia.org/wiki/Median_absolute_deviation.' function: processor.RobustZScoreNorm - docstring: null function: processor.fit - docstring: ' Cross Sectional ZScore Normalization self.fields_group = fields_groupif method == "zscore":self.zscore_func = zscoreelif method == "robust":self.zscore_func = robust_zscoreelse:raise NotImplementedError(f"This type of input is not supported")# try not modify original dataframeif not isinstance(self.fields_group, list):self.fields_group = [self.fields_group]for g in self.fields_group:cols = get_group_columns(df, g)df[cols] = df[cols].groupby("datetime", group_keys=False).apply(self.zscore_func)return df' function: processor.CSZScoreNorm - docstring: " \nCross Sectional Rank Normalization.\"Cross Sectional\" is often\ \ used to describe data operations.The operations across different stocks are\ \ often called Cross Sectional Operation.For example, CSRankNorm is an operation\ \ that grouping the data by each day and rank `across` all the stocks in each\ \ day.Explanation about 3.46 & 0.5.. code-block:: pythonimport numpy as npimport\ \ pandas as pdx = np.random.random(10000) # for any variablex_rank = pd.Series(x).rank(pct=True)\ \ # if it is converted to rank, it will be a uniform distributedx_rank_norm =\ \ (x_rank - x_rank.mean()) / x_rank.std() # Normally, we will normalize it to\ \ make it like normal distributionx_rank.mean() # accounts for 0.51 / x_rank.std()\ \ # accounts for 3.46" function: processor.CSRankNorm - docstring: ' Cross Sectional Fill Nan self.fields_group = fields_groupcols = get_group_columns(df, self.fields_group)df[cols] = df[cols].groupby("datetime", group_keys=False).apply(lambda x: x.fillna(x.mean()))return df' function: processor.CSZFillna - docstring: ' Process the storage of from df into hasing stock format from .storage import HashingStockStorage # pylint: disable=C0415return HashingStockStorage.from_df(df)' function: processor.HashStockFormat - docstring: " \nThis is a filter to filter stock.Only keep the data that exist\ \ from start_time to end_time (the existence in the middle is not checked.)WARNING:\ \ It may induce leakage!!!" function: processor.TimeRangeFlt - docstring: " \nPreparing data for model training and inferencing." function: __init__.Dataset - docstring: " \nconfig is designed to configure and parameters that cannot\ \ be learned from the data" function: __init__.config - docstring: " \nSetup the data.We split the setup_data function for following\ \ situation:- User have a Dataset object with learned status on disk.- User load\ \ the Dataset object from the disk.- User call `setup_data` to load new data.-\ \ User prepare data for model based on previous status." function: __init__.setup_data - docstring: " \nThe type of dataset depends on the model. (It could be pd.DataFrame,\ \ pytorch.DataLoader, etc.)The parameters should specify the scope for the prepared\ \ dataThe method should:- process the data- return the processed dataReturns-------object:return\ \ the object" function: __init__.prepare - docstring: " \nDataset with Data(H)andlerUser should try to put the data preprocessing\ \ functions into handler.Only following data processing functions should be placed\ \ in Dataset:- The processing is related to specific model.- The processing is\ \ related to data split." function: __init__.DatasetH - docstring: " \nInitialize the DatasetHParameters----------handler_kwargs\ \ : dictConfig of DataHandler, which could include the following arguments:- arguments\ \ of DataHandler.conf_data, such as 'instruments', 'start_time' and 'end_time'.kwargs\ \ : dictConfig of DatasetH, such as- segments : dictConfig of segments which is\ \ same as 'segments' in self.__init__" function: __init__.config - docstring: " \nSetup the DataParameters----------handler_kwargs : dictinit\ \ arguments of DataHandler, which could include the following arguments:- init_type\ \ : Init Type of Handler- enable_cache : whether to enable cache" function: __init__.setup_data - docstring: " \nGive a query, retrieve the according dataParameters----------slc\ \ : please refer to the docs of `prepare`NOTE: it may not be an instance of slice.\ \ It may be a segment of `segments` from `def prepare`" function: __init__._prepare_seg - docstring: " \nPrepare the data for learning and inference.Parameters----------segments\ \ : Union[List[Text], Tuple[Text], Text, slice]Describe the scope of the data\ \ to be preparedHere are some examples:- 'train'- ['train', 'valid']col_set :\ \ strThe col_set will be passed to self.handler when fetching data.TODO: make\ \ it automatic:- select DK_I for test data- select DK_L for training data.data_key\ \ : strThe data to fetch: DK_*Default is DK_I, which indicate fetching data for\ \ **inference**.kwargs :The parameters that kwargs may contain:flt_col : strIt\ \ only exists in TSDatasetH, can be used to add a column of data(True or False)\ \ to filter data.This parameter is only supported when it is an instance of TSDatasetH.Returns-------Union[List[pd.DataFrame],\ \ pd.DataFrame]:Raises------NotImplementedError:" function: __init__.prepare - docstring: null function: __init__.get_min_time - docstring: null function: __init__.get_max_time - docstring: ' it will act like sort and return the max value or None candidate = Nonefor k, seg in segments.items():point = seg[idx]if point is None:# None indicates unbounded, return directlyreturn Noneelif candidate is None or cmp(key_func(candidate), key_func(point)):candidate = pointreturn candidate' function: __init__._get_extrema - docstring: " \n(T)ime-(S)eries DataSamplerThis is the result of TSDatasetHIt\ \ works like `torch.data.utils.Dataset`, it provides a very convenient interface\ \ for constructing time-seriesdataset based on tabular data.- On time step dimension,\ \ the smaller index indicates the historical data and the larger index indicates\ \ the futuredata.If user have further requirements for processing data, user could\ \ process them based on `TSDataSampler` or createmore powerful subclasses.Known\ \ Issues:- For performance issues, this Sampler will convert dataframe into arrays\ \ for better performance. This could resultin a different data typeIndices design:TSDataSampler\ \ has a index mechanism to help users query time-series data efficiently.The definition\ \ of related variables:data_arr: np.ndarrayThe original data. it will contains\ \ all the original data.The querying are often for time-series of a specific stock.By\ \ leveraging this data charactoristics to speed up querying, the multi-index of\ \ data_arr is rearranged in (instrument, datetime) orderdata_index: pd.MultiIndex\ \ with index order it has the same shape with `idx_map`.\ \ Each elements of them are expected to be aligned.idx_map: np.ndarrayIt is the\ \ indexable data. It originates from data_arr, and then filtered by 1) `start`\ \ and `end` 2) `flt_data`The extra data in data_arr is useful in following cases1)\ \ creating meaningful time series data before `start` instead of padding them\ \ with zeros2) some data are excluded by `flt_data` (e.g. no sample pair\ \ for that index). but they are still used in time-series in XFinnally, it will\ \ look like.array([[ 0, 0],[ 1, 0],[ 2, 0],...,[241, 348],[242, 348],[243,\ \ 348]], dtype=int32)It list all indexable data(some data only used in historical\ \ time series data may not be indexabla), the values are the corresponding row\ \ and col in idx_dfidx_df: pd.DataFrameIt aims to map the \ \ key to the original position in data_arrFor example, it may look like (NOTE:\ \ the index for a instrument time-series is continoues in memory)instrument SH600000\ \ SH600008 SH600009 SH600010 SH600011 SH600015 ...datetime2017-01-03 0\ \ 242 473 717 NaN 974 ...2017-01-04 1 243\ \ 474 718 NaN 975 ...2017-01-05 2 244 475\ \ 719 NaN 976 ...2017-01-06 3 245 476 720\ \ NaN 977 ...With these two indices(idx_map, idx_df) and original data(data_arr),\ \ we can make the following queries fast (implemented in __getitem__)(1) Get the\ \ i-th indexable sample(time-series): (indexable sample index) -> [idx_map]\ \ -> (row col) -> [idx_df] -> (index in data_arr)(2) Get the specific sample by\ \ : (, i.e. ) -> [idx_df]\ \ -> (index in data_arr)(3) Get the index of a time-series data: (get the , refer to (1), (2)) -> [idx_df] -> (all indices in data_arr for time-series)" function: '__init__.TSDataSampler:' - docstring: null function: __init__.slice_idx_map_and_data_index - docstring: null function: __init__.idx_map2arr - docstring: null function: __init__.flt_idx_map - docstring: " \nGet the pandas index of the data, it will be useful in following\ \ scenarios- Special sampler will be used (e.g. user want to sample day by day)" function: __init__.get_index - docstring: null function: __init__.config - docstring: " \nThe relation of the dataParameters----------data : pd.DataFrameA\ \ DataFrame with index in order RSQR5 RESI5 WVMA5\ \ LABEL0instrument datetimeSH600000 2017-01-03 0.016389 0.461632 -1.154788\ \ -0.0480562017-01-04 0.884545 -0.110597 -1.059332 -0.0301392017-01-05 0.507540\ \ -0.535493 -1.099665 -0.6449832017-01-06 -1.267771 -0.669685 -1.636733 0.2953662017-01-09\ \ 0.339346 0.074317 -0.984989 0.765540Returns-------Tuple[pd.DataFrame, dict]:1)\ \ the first element: reshape the original index into a \ \ 2D dataframeinstrument SH600000 SH600008 SH600009 SH600010 SH600011 SH600015\ \ ...datetime2017-01-03 0 242 473 717 NaN 974\ \ ...2017-01-04 1 243 474 718 NaN 975 ...2017-01-05\ \ 2 244 475 719 NaN 976 ...2017-01-06 \ \ 3 245 476 720 NaN 977 ...2) the second element: {: }" function: __init__.build_index - docstring: null function: __init__.empty - docstring: " \nget series indices of self.data_arr from the row, col indices\ \ of self.idx_dfParameters----------row : intthe row in self.idx_dfcol : intthe\ \ col in self.idx_dfReturns-------np.array:The indices of data of the data" function: __init__._get_indices - docstring: " \nget the col index and row index of a given sample index in\ \ self.idx_dfParameters----------idx :the input of `__getitem__`Returns-------Tuple[int]:the\ \ row and col index" function: __init__._get_row_col - docstring: " \n(T)ime-(S)eries Dataset (H)andlerConvert the tabular data to Time-Series\ \ dataRequirements analysisThe typical workflow of a user to get time-series data\ \ for an sample- process features- slice proper data from data handler: dimension\ \ of sample - Build relation of samples by index-\ \ Be able to sample times series of data - It will be better\ \ if the interface is like \"torch.utils.data.Dataset\"- User could build customized\ \ batch based on the data- The dimension of a batch of data " function: __init__.TSDatasetH - docstring: null function: __init__.config - docstring: null function: __init__.setup_data - docstring: null function: __init__._extend_slice - docstring: " \nsplit the _prepare_raw_seg is to leave a hook for data preprocessing\ \ before creating processing dataNOTE: TSDatasetH only support slc segment on\ \ datetime !!!" function: __init__._prepare_seg - docstring: " \nTo initialize the Reweighter, users should provide specific\ \ methods to let reweighter do the reweighting (such as sample-wise, rule-based)." function: 'weight.Reweighter:' - docstring: " \nGet weights for dataParameters----------data : objectThe input\ \ data.The first dimension is the index of samplesReturns-------object:the weights\ \ info for the data" function: weight.reweight - docstring: " \nBase data storage for datahandler- pd.DataFrame is the default\ \ data storage format in Qlib datahandler- If users want to use custom data storage,\ \ they should define subclass inherited BaseHandlerStorage, and implement the\ \ following method" function: 'storage.BaseHandlerStorage:' - docstring: ' fetch data from the data storage Parameters----------selector : Union[pd.Timestamp, slice, str]describe how to select data by indexlevel : Union[str, int]which index level to select the data- if level is None, apply selector to df directlycol_set : Union[str, List[str]]- if isinstance(col_set, str):select a set of meaningful columns.(e.g. features, columns)if col_set == DataHandler.CS_RAW:the raw dataset will be returned.- if isinstance(col_set, List[str]):select several sets of meaningful columns, the returned data has multiple levelfetch_orig : boolReturn the original data instead of copy if possible.proc_func: Callableplease refer to the doc of DataHandler.fetchReturns-------pd.DataFramethe dataframe fetched' function: storage.fetch - docstring: null function: storage.from_df - docstring: ' whether the arg `proc_func` in `fetch` method is supported. raise NotImplementedError("is_proc_func_supported method is not implemented!")' function: storage.is_proc_func_supported - docstring: ' Hashing data storage for datahanlder - The default data storage pandas.DataFrame is too slow when randomly accessing one stock''s data- HashingStockStorage hashes the multiple stocks'' data(pandas.DataFrame) by the key `stock_id`.- HashingStockStorage hashes the pandas.DataFrame into a dict, whose key is the stock_id(str) and value this stock data(panda.DataFrame), it has the following format:{stock1_id: stock1_data,stock2_id: stock2_data,...stockn_id: stockn_data,}- By the `fetch` method, users can access any stock data with much lower time cost than default data storage' function: storage.HashingStockStorage - docstring: null function: storage.from_df - docstring: ' fetch the data with stock selector Parameters----------selector : Union[pd.Timestamp, slice, str]describe how to select data by indexlevel : Union[str, int]which index level to select the data- if level is None, apply selector to df directly- the `_fetch_hash_df_by_stock` will parse the stock selector in arg `selector`Returns-------DictThe dict whose key is stock_id, value is the stock''s data' function: storage._fetch_hash_df_by_stock - docstring: null function: storage.fetch - docstring: ' Risk Analysis NOTE:The calculation of annulaized return is different from the definition of annualized return.It is implemented by design.Qlib tries to cumulated returns by summation instead of production to avoid the cumulated curve being skewed exponentially.All the calculation of annualized returns follows this principle in Qlib.TODO: add a parameter to enable calculating metrics with production accumulation of return.Parameters----------r : pandas.Seriesdaily return series.N: intscaler for annualizing information_ratio (day: 252, week: 50, month: 12), at least one of `N` and `freq` should existfreq: stranalysis frequency used for calculating the scaler, at least one of `N` and `freq` should exist' function: evaluate.risk_analysis - docstring: null function: evaluate.cal_risk_analysis_scaler - docstring: ' analyze statistical time-series indicators of trading Parameters----------df : pandas.DataFramecolumns: like [''pa'', ''pos'', ''ffr'', ''deal_amount'', ''value''].Necessary fields:- ''pa'' is the price advantage in trade indicators- ''pos'' is the positive rate in trade indicators- ''ffr'' is the fulfill rate in trade indicatorsOptional fields:- ''deal_amount'' is the total deal deal_amount, only necessary when method is ''amount_weighted''- ''value'' is the total trade value, only necessary when method is ''value_weighted''index: Index(datetime)method : str, optionalstatistics method of pa/ffr, by default "mean"- if method is ''mean'', count the mean statistical value of each trade indicator- if method is ''amount_weighted'', count the deal_amount weighted mean statistical value of each trade indicator- if method is ''value_weighted'', count the value weighted mean statistical value of each trade indicatorNote: statistics method of pos is always "mean"Returns-------pd.DataFramestatistical value of each trade indicators' function: evaluate.indicator_analysis - docstring: ' initialize the strategy and executor, then executor the backtest of daily frequency Parameters----------start_time : Union[str, pd.Timestamp]closed start time for backtest**NOTE**: This will be applied to the outmost executor''s calendar.end_time : Union[str, pd.Timestamp]closed end time for backtest**NOTE**: This will be applied to the outmost executor''s calendar.E.g. Executor[day](Executor[1min]), setting `end_time == 20XX0301` will include all the minutes on 20XX0301strategy : Union[str, dict, BaseStrategy]for initializing outermost portfolio strategy. Please refer to the docs of init_instance_by_config for more information.E.g... code-block:: python# dictstrategy = {"class": "TopkDropoutStrategy","module_path": "qlib.contrib.strategy.signal_strategy","kwargs": {"signal": (model, dataset),"topk": 50,"n_drop": 5,},}# BaseStrategypred_score = pd.read_pickle("score.pkl")["score"]STRATEGY_CONFIG = {"topk": 50,"n_drop": 5,"signal": pred_score,}strategy = TopkDropoutStrategy(**STRATEGY_CONFIG)# str example.# 1) specify a pickle object# - path like ''file:////obj.pkl''# 2) specify a class name# - "ClassName": getattr(module, "ClassName")() will be used.# 3) specify module path with class name# - "a.b.c.ClassName" getattr(, "ClassName")() will be used.executor : Union[str, dict, BaseExecutor]for initializing the outermost executor.benchmark: strthe benchmark for reporting.account : Union[float, int, Position]information for describing how to creating the accountFor `float` or `int`:Using Account with only initial cashFor `Position`:Using Account with a Positionexchange_kwargs : dictthe kwargs for initializing ExchangeE.g... code-block:: pythonexchange_kwargs = {"freq": freq,"limit_threshold": None, # limit_threshold is None, using C.limit_threshold"deal_price": None, # deal_price is None, using C.deal_price"open_cost": 0.0005,"close_cost": 0.0015,"min_cost": 5,}pos_type : strthe type of Position.Returns-------report_normal: pd.DataFramebacktest reportpositions_normal: pd.DataFramebacktest positions' function: evaluate.backtest_daily - docstring: " \nA backtest for long-short strategy:param pred: The trading\ \ signal produced on day `T`.:param topk: The short topk securities and\ \ long topk securities.:param deal_price: The price to deal the trading.:param\ \ shift: Whether to shift prediction by one day. The trading day will be\ \ T+1 if shift==1.:param open_cost: open transaction cost.:param close_cost:\ \ close transaction cost.:param trade_unit: 100 for China A.:param limit_threshold:\ \ limit move 0.1 (10%) for example, long and short with same limit.:param min_cost:\ \ min transaction cost.:param subscribe_fields: subscribe fields.:param extract_codes:\ \ bool.will we pass the codes extracted from the pred to the exchange.NOTE: This\ \ will be faster with offline qlib.:return: The result of backtest,\ \ it is represented by a dict.{ \"long\": long_returns(excess),\"short\": short_returns(excess),\"\ long_short\": long_short_returns}" function: evaluate.long_short_backtest - docstring: null function: 'analyzer.AnalyzerTemp:' - docstring: " \nIt behaves the same as self.recorder.load_object.But it is\ \ an easier interface because users don't have to care about `get_path` and `artifact_path`Parameters----------name\ \ : strthe name for the file to be load.Return------The stored records." function: analyzer.load - docstring: " \nAnalyse data index, distribution .etcParameters----------Return------The\ \ handled data." function: analyzer.analyse - docstring: " \nThis is the Signal Analysis class that generates the analysis\ \ results such as IC and IR.default output image filename is \"HFAnalyzerTable.jpeg\"" function: analyzer.HFAnalyzer - docstring: null function: analyzer.analyse - docstring: " \nThis is the Signal Analysis class that generates the analysis\ \ results such as IC and IR.default output image filename is \"signalAnalysis.jpeg\"" function: analyzer.SignalAnalyzer - docstring: ' Get position value by existed close data df close_data_df:pd.DataFramemulti-indexclose_data_df[''$close''][stock_id][evaluate_date]: close price for (stock_id, evaluate_date)position:same in get_position_value()' function: evaluate_portfolio._get_position_value_from_df - docstring: ' sum of close*amount get value of positionuse close pricepositions:{Timestamp(''2016-01-05 00:00:00''):{''SH600022'':{''amount'':100.00,''price'':12.00},''cash'':100000.0}}It means Hold 100.0 ''SH600022'' and 100000.0 RMB in ''2016-01-05''' function: evaluate_portfolio.get_position_value - docstring: null function: evaluate_portfolio.get_position_list_value - docstring: ' Parameters generate daily return series from position viewpositions: positions generated by strategyinit_asset_value : init asset valuereturn: pd.Series of daily return , return_series[date] = daily return rate' function: evaluate_portfolio.get_daily_return_series_from_positions - docstring: ' Annualized Returns p_r = (p_end / p_start)^{(250/n)} - 1p_r annual returnp_end final valuep_start init valuen days of backtest' function: evaluate_portfolio.get_annual_return_from_positions - docstring: ' Risk Analysis from daily return series Parameters----------r : pandas.Seriesdaily return seriesmethod : strinterest calculation method, ci(compound interest)/si(simple interest)' function: evaluate_portfolio.get_annaul_return_from_return_series - docstring: ' Risk Analysis Parameters----------r : pandas.Seriesdaily return seriesmethod : strinterest calculation method, ci(compound interest)/si(simple interest)risk_free_rate : floatrisk_free_rate, default as 0.00, can set as 0.03 etc' function: evaluate_portfolio.get_sharpe_ratio_from_return_series - docstring: ' Risk Analysis from asset value cumprod wayParameters----------r : pandas.Seriesdaily return series' function: evaluate_portfolio.get_max_drawdown_from_series - docstring: null function: evaluate_portfolio.get_turnover_rate - docstring: ' Risk Analysis beta Parameters----------r : pandas.Seriesdaily return series of strategyb : pandas.Seriesdaily return series of baseline' function: evaluate_portfolio.get_beta - docstring: null function: evaluate_portfolio.get_alpha - docstring: null function: evaluate_portfolio.get_volatility_from_series - docstring: ' Rank IC Parameters----------r : pandas.Seriesdaily score series of featureb : pandas.Seriesdaily return series' function: evaluate_portfolio.get_rank_ic - docstring: " \nThis model will load a score file, and return score at date exists\ \ in score file." function: online_model.ScoreFileModel - docstring: null function: online_model.get_data_with_date - docstring: null function: online_model.predict - docstring: null function: online_model.score - docstring: null function: online_model.fit - docstring: " \nload a pickle fileParameterfile_path : string / pathlib.Path()path\ \ of file to be loaded:returnAn instance loaded from file" function: utils.load_instance - docstring: " \nsave(dump) an instance to a pickle fileParameterinstance :data\ \ to be dumpedfile_path : string / pathlib.Path()path of file to be dumped" function: utils.save_instance - docstring: null function: utils.create_user_folder - docstring: " \n1. Get the dates that need to do trading till today for user {user_id}dates[0]\ \ indicate the latest trading date of User{user_id},if User{user_id} haven't do\ \ trading before, than dates[0] presents the init date of User{user_id}.2. Set\ \ the exchange with exchange_config fileParameterum : UserManager()today : pd.Timestamp()user_id\ \ : str:returndates : list of pd.Timestamptrade_exchange : Exchange()" function: utils.prepare - docstring: " \nThis module is designed to manager the users in online systemall\ \ users' data were assumed to be saved in user_data_pathParameteruser_data_path\ \ : stringdata path that all users' data were saved invariables:data_path : stringdata\ \ path that all users' data were saved inusers_file : stringA path of the file\ \ record the add_date of userssave_report : boolwhether to save report after each\ \ trading processusers : dict{}[user_id]->User()the python dict save instances\ \ of User() for each user_iduser_record : pd.Dataframeuser_id(string), add_date(string)indicate\ \ the add_date for each users" function: 'manager.UserManager:' - docstring: " \nload all users' data into manager" function: manager.load_users - docstring: " \nreturn a instance of User() represents a user to be processedParameteruser_id\ \ : string:returnuser : User()" function: manager.load_user - docstring: " \nsave a instance of User() to user data pathParameteruser_id\ \ : string" function: manager.save_user_data - docstring: " \nadd the new user {user_id} into user datawill create a new\ \ folder named \"{user_id}\" in user data pathParameteruser_id : stringinit_cash\ \ : intconfig_file : str/pathlib.Path()path of config file" function: manager.add_user - docstring: " \nremove user {user_id} in current user datasetwill delete the\ \ folder \"{user_id}\" in user data path:paramuser_id : string" function: manager.remove_user - docstring: " \nParameters----------client: strThe qlib client config file(.yaml)" function: 'operator.Operator:' - docstring: ' Initial UserManager(), get predict date and trade date Parameters----------client: strThe qlib client config file(.yaml)path : strPath to save user account.date : str (YYYY-MM-DD)Trade date, when the generated order list will be traded.Return----------um: UserManager()pred_date: pd.Timestamptrade_date: pd.Timestamp' function: operator.init - docstring: ' Add a new user into the a folder to run ''online'' module. Parameters----------id : strUser id, should be unique.config : strThe file path (yaml) of user configpath : strPath to save user account.date : str (YYYY-MM-DD)The date that user account was added.' function: operator.add_user - docstring: ' Remove user from folder used in ''online'' module. Parameters----------id : strUser id, should be unique.path : strPath to save user account.' function: operator.remove_user - docstring: ' Generate order list that will be traded at ''date''. Parameters----------date : str (YYYY-MM-DD)Trade date, when the generated order list will be traded.path : strPath to save user account.' function: operator.generate - docstring: ' Execute the orderlist at ''date''. Parameters----------date : str (YYYY-MM-DD)Trade date, that the generated order list will be traded.exchange_config: strThe file path (yaml) of exchange configpath : strPath to save user account.' function: operator.execute - docstring: ' Update account at ''date''. Parameters----------date : str (YYYY-MM-DD)Trade date, that the generated order list will be traded.path : strPath to save user account.type : strwhich executor was been used to execute the order list''SIM'': SimulatorExecutor()' function: operator.update - docstring: ' Run the ( generate_trade_decision -> execute_order_list -> update_account) process everyday from start date to end date.Parameters----------id : struser id, need to be uniqueconfig : strThe file path (yaml) of user configexchange_config: strThe file path (yaml) of exchange configstart : str "YYYY-MM-DD"The start date to run the online simulateend : str "YYYY-MM-DD"The end date to run the online simulatepath : strPath to save user account.bench : strThe benchmark that our result compared with.''SH000905'' for csi500, ''SH000300'' for csi300' function: operator.simulate - docstring: ' show the newly report (mean, std, information_ratio, annualized_return) Parameters----------id : struser id, need to be uniquepath : strPath to save user account.bench : strThe benchmark that our result compared with.''SH000905'' for csi500, ''SH000300'' for csi300' function: operator.show - docstring: " \nWill be called in online moduleneed to return the data that\ \ used to predict the label (score) of stocks at date.:paramdate: pd.Timestamppredict\ \ date:return:data: the input data that used to predict the label (score) of stocks\ \ at predict date." function: __init__.get_data_with_date - docstring: " \nA user in online system, which contains account, strategy\ \ and model three module.Parameteraccount : Account()strategy :a strategy instancemodel\ \ :a model instancereport_save_path : stringthe path to save report. Will not\ \ save report if Noneverbose : boolWhether to print the info during the process" function: 'user.User:' - docstring: " \ninit state when each trading date beginParameterdate : pd.Timestamp" function: user.init_state - docstring: " \nreturn the latest trading date for user {user_id}Parameteruser_id\ \ : string:returndate : string (e.g '2018-10-08')" function: user.get_latest_trading_date - docstring: " \nshow the newly report (mean, std, information_ratio, annualized_return)Parameterbenchmark\ \ : stringbench that to be compared, 'SH000905' for csi500" function: user.showReport - docstring: null function: 'order_generator.OrderGenerator:' - docstring: ' generate_order_list_from_target_weight_position :param current: The current position:type current: Position:param trade_exchange::type trade_exchange: Exchange:param target_weight_position: {stock_id : weight}:type target_weight_position: dict:param risk_degree::type risk_degree: float:param pred_start_time::type pred_start_time: pd.Timestamp:param pred_end_time::type pred_end_time: pd.Timestamp:param trade_start_time::type trade_start_time: pd.Timestamp:param trade_end_time::type trade_end_time: pd.Timestamp:rtype: list' function: order_generator.generate_order_list_from_target_weight_position - docstring: ' Order Generator With Interact ' function: order_generator.OrderGenWInteract - docstring: ' generate_order_list_from_target_weight_position No adjustment for for the nontradable share.All the tadable value is assigned to the tadable stock according to the weight.if interact == True, will use the price at trade date to generate order listelse, will only use the price before the trade date to generate order list:param current::type current: Position:param trade_exchange::type trade_exchange: Exchange:param target_weight_position::type target_weight_position: dict:param risk_degree::type risk_degree: float:param pred_start_time::type pred_start_time: pd.Timestamp:param pred_end_time::type pred_end_time: pd.Timestamp:param trade_start_time::type trade_start_time: pd.Timestamp:param trade_end_time::type trade_end_time: pd.Timestamp:rtype: list' function: order_generator.generate_order_list_from_target_weight_position - docstring: ' Order Generator Without Interact ' function: order_generator.OrderGenWOInteract - docstring: ' generate_order_list_from_target_weight_position generate order list directly not using the information (e.g. whether can be traded, the accurate trade price)at trade date.In target weight position, generating order list need to know the price of objective stock in trade date,but we cannot get thatvalue when do not interact with exchange, so we check the %close price at pred_date or price recordedin current position.:param current::type current: Position:param trade_exchange::type trade_exchange: Exchange:param target_weight_position::type target_weight_position: dict:param risk_degree::type risk_degree: float:param pred_start_time::type pred_start_time: pd.Timestamp:param pred_end_time::type pred_end_time: pd.Timestamp:param trade_start_time::type trade_start_time: pd.Timestamp:param trade_end_time::type trade_end_time: pd.Timestamp:rtype: list of generated orders' function: order_generator.generate_order_list_from_target_weight_position - docstring: " \nParameters-----------signal :the information to describe a\ \ signal. Please refer to the docs of `qlib.backtest.signal.create_signal_from`the\ \ decision of the strategy will base on the given signalrisk_degree : floatposition\ \ percentage of total value.trade_exchange : Exchangeexchange that provides market\ \ info, used to deal order and generate report- If `trade_exchange` is None, self.trade_exchange\ \ will be set with common_infra- It allowes different trade_exchanges is used\ \ in different executions.- For example:- In daily execution, both daily exchange\ \ and minutely are usable, but the daily exchange is recommended because it runs\ \ faster.- In minutely execution, the daily exchange is not usable, only the minutely\ \ exchange is recommended." function: signal_strategy.BaseSignalStrategy - docstring: ' get_risk_degree Return the proportion of your total value you will use in investment.Dynamically risk_degree will result in Market timing.' function: signal_strategy.get_risk_degree - docstring: " \nParameters-----------topk : intthe number of stocks in the\ \ portfolio.n_drop : intnumber of stocks to be replaced in each trading date.method_sell\ \ : strdropout method_sell, random/bottom.method_buy : strdropout method_buy,\ \ random/top.hold_thresh : intminimum holding daysbefore sell stock , will check\ \ current.get_stock_count(order.stock_id) >= self.hold_thresh.only_tradable :\ \ boolwill the strategy only consider the tradable stock when buying and selling.if\ \ only_tradable:strategy will make decision with the tradable state of the stock\ \ info and avoid buy and sell them.else:strategy will make buy sell decision without\ \ checking the tradable state of the stock.forbid_all_trade_at_limit : boolif\ \ forbid all trades when limit_up or limit_down reached.if forbid_all_trade_at_limit:strategy\ \ will not do any trade when price reaches limit up/down, even not sell at limit\ \ up nor buy atlimit down, though allowed in reality.else:strategy will sell at\ \ limit up and buy ad limit down." function: signal_strategy.TopkDropoutStrategy - docstring: null function: signal_strategy.generate_trade_decision - docstring: null function: signal_strategy.get_first_n - docstring: null function: signal_strategy.get_last_n - docstring: null function: signal_strategy.filter_stock - docstring: null function: signal_strategy.get_first_n - docstring: null function: signal_strategy.get_last_n - docstring: null function: signal_strategy.filter_stock - docstring: " \nsignal :the information to describe a signal. Please refer\ \ to the docs of `qlib.backtest.signal.create_signal_from`the decision of the\ \ strategy will base on the given signaltrade_exchange : Exchangeexchange that\ \ provides market info, used to deal order and generate report- If `trade_exchange`\ \ is None, self.trade_exchange will be set with common_infra- It allowes different\ \ trade_exchanges is used in different executions.- For example:- In daily execution,\ \ both daily exchange and minutely are usable, but the daily exchange is recommended\ \ because it runs faster.- In minutely execution, the daily exchange is not usable,\ \ only the minutely exchange is recommended." function: signal_strategy.WeightStrategyBase - docstring: " \nGenerate target position from score for this date and the\ \ current position.The cash is not considered in the positionParameters-----------score\ \ : pd.Seriespred score for this trade date, index is stock_id, contain 'score'\ \ column.current : Position()current position.trade_start_time: pd.Timestamptrade_end_time:\ \ pd.Timestamp" function: signal_strategy.generate_target_weight_position - docstring: null function: signal_strategy.generate_trade_decision - docstring: " Enhanced Indexing Strategy\nEnhanced indexing combines the arts\ \ of active management and passive management,with the aim of outperforming a\ \ benchmark index (e.g., S&P 500) in terms ofportfolio return while controlling\ \ the risk exposure (a.k.a. tracking error).Users need to prepare their risk model\ \ data like below:.. code-block:: text\u251C\u2500\u2500 /path/to/riskmodel\u251C\ \u2500\u2500\u2500\u2500 20210101\u251C\u2500\u2500\u2500\u2500\u2500\u2500 factor_exp.{csv|pkl|h5}\u251C\ \u2500\u2500\u2500\u2500\u2500\u2500 factor_cov.{csv|pkl|h5}\u251C\u2500\u2500\ \u2500\u2500\u2500\u2500 specific_risk.{csv|pkl|h5}\u251C\u2500\u2500\u2500\u2500\ \u2500\u2500 blacklist.{csv|pkl|h5} # optionalThe risk model data can be obtained\ \ from risk data provider. You can also use`qlib.model.riskmodel.structured.StructuredCovEstimator`\ \ to prepare these data.Args:riskmodel_path (str): risk model pathname_mapping\ \ (dict): alternative file names" function: signal_strategy.EnhancedIndexingStrategy - docstring: null function: signal_strategy.get_risk_data - docstring: ' TWAP Strategy for trading NOTE:- This TWAP strategy will celling round when trading. This will make the TWAP trading strategy produce the orderearlier when the total trade unit of amount is less than the trading step' function: rule_strategy.TWAPStrategy - docstring: " \nParameters----------outer_trade_decision : BaseTradeDecision,\ \ optional" function: rule_strategy.reset - docstring: null function: rule_strategy.generate_trade_decision - docstring: " \n(S)elect the (B)etter one among every two adjacent trading (B)ars\ \ to sell or buy." function: rule_strategy.SBBStrategyBase - docstring: " \nParameters----------outer_trade_decision : BaseTradeDecision,\ \ optional" function: rule_strategy.reset - docstring: null function: rule_strategy._pred_price_trend - docstring: null function: rule_strategy.generate_trade_decision - docstring: " \n(S)elect the (B)etter one among every two adjacent trading (B)ars\ \ to sell or buy with (EMA) signal." function: rule_strategy.SBBStrategyEMA - docstring: null function: rule_strategy._reset_signal - docstring: " \nreset level-shared infra- After reset the trade calendar,\ \ the signal will be changed" function: rule_strategy.reset_level_infra - docstring: null function: rule_strategy._pred_price_trend - docstring: " \nParameters----------instruments : Union[List, str], optionalinstruments\ \ of Volatility, by default \"csi300\"freq : str, optionalfreq of Volatility,\ \ by default \"day\"Note: `freq` may be different from `time_per_step`" function: rule_strategy.ACStrategy - docstring: null function: rule_strategy._reset_signal - docstring: " \nreset level-shared infra- After reset the trade calendar,\ \ the signal will be changed" function: rule_strategy.reset_level_infra - docstring: " \nParameters----------outer_trade_decision : BaseTradeDecision,\ \ optional" function: rule_strategy.reset - docstring: null function: rule_strategy.generate_trade_decision - docstring: " \nParameters----------trade_range : Tupleplease refer to the\ \ `trade_range` parameter of BaseStrategysample_ratio : floatthe ratio of all\ \ orders are sampledvolume_ratio : floatthe volume of the total dayraito of the\ \ total volume of a specific daymarket : strstock pool for sampling" function: rule_strategy.RandomOrderStrategy - docstring: null function: rule_strategy.generate_trade_decision - docstring: " \nMotivation:- This class provides an interface for user to read\ \ orders from csv files." function: rule_strategy.FileOrderStrategy - docstring: " \nParameters----------execute_result :execute_result will be\ \ ignored in FileOrderStrategy" function: rule_strategy.generate_trade_decision - docstring: " \nParameters----------topk : inttop-N stocks to buyrisk_degree\ \ : floatposition percentage of total value buy_method:rank_fill: assign the weight\ \ stocks that rank high first(1/topk max)average_fill: assign the weight to the\ \ stocks rank high averagely." function: cost_control.SoftTopkStrategy - docstring: ' get_risk_degree Return the proportion of your total value you will used in investment.Dynamically risk_degree will result in Market timing' function: cost_control.get_risk_degree - docstring: " \nParameters----------score:pred score for this trade date,\ \ pd.Series, index is stock_id, contain 'score' columncurrent:current position,\ \ use Position() classtrade_date:trade dategenerate target position from score\ \ for this date and the current positionThe cache is not considered in the position" function: cost_control.generate_target_weight_position - docstring: ' Portfolio Optimizer The following optimization algorithms are supported:- `gmv`: Global Minimum Variance Portfolio- `mvo`: Mean Variance Optimized Portfolio- `rp`: Risk Parity- `inv`: Inverse VolatilityNote:This optimizer always assumes full investment and no-shorting.' function: optimizer.PortfolioOptimizer - docstring: null function: optimizer._optimize - docstring: ' Inverse volatility vola = np.diag(S) ** 0.5w = 1 / volaw /= w.sum()return w' function: optimizer._optimize_inv - docstring: ' optimize global minimum variance portfolio This method solves the following optimization problemmin_w w'' S ws.t. w >= 0, sum(w) == 1where `S` is the covariance matrix.' function: optimizer._optimize_gmv - docstring: ' optimize mean-variance portfolio This method solves the following optimization problemmin_w - w'' r + lamb * w'' S ws.t. w >= 0, sum(w) == 1where `S` is the covariance matrix, `u` is the expected returns,and `lamb` is the risk aversion parameter.' function: optimizer._optimize_mvo - docstring: ' optimize risk parity portfolio This method solves the following optimization problemmin_w sum_i [w_i - (w'' S w) / ((S w)_i * N)]**2s.t. w >= 0, sum(w) == 1where `S` is the covariance matrix and `N` is the number of stocks.' function: optimizer._optimize_rp - docstring: ' global minimum variance optimization objective Optimization objectivemin_w w'' S w' function: optimizer._get_objective_gmv - docstring: null function: optimizer.func - docstring: ' mean-variance optimization objective Optimization objectivemin_w - w'' r + lamb * w'' S w' function: optimizer._get_objective_mvo - docstring: null function: optimizer.func - docstring: ' risk-parity optimization objective Optimization objectivemin_w sum_i [w_i - (w'' S w) / ((S w)_i * N)]**2' function: optimizer._get_objective_rp - docstring: null function: optimizer.func - docstring: ' optimization constraints Defines the following constraints:- no shorting and leverage: 0 <= w <= 1- full investment: sum(w) == 1- turnover constraint: |w - w0| <= delta' function: optimizer._get_constrains - docstring: ' solve optimization Args:n (int): number of parametersobj (callable): optimization objectivebounds (Bounds): bounds of parameterscons (list): optimization constraints' function: optimizer._solve - docstring: ' Construct portfolio with a optimization related method @abc.abstractmethodGenerate a optimized portfolio allocation' function: base.BaseOptimizer - docstring: " \nPortfolio Optimizer for Enhanced IndexingNotations:w0: current\ \ holding weightswb: benchmark weightr: expected returnF: factor exposurecov_b:\ \ factor covariancevar_u: residual variance (diagonal)lamb: risk aversion parameterdelta:\ \ total turnover limitb_dev: benchmark deviation limitf_dev: factor deviation\ \ limitAlso denote:d = w - wb: benchmark deviationv = d @ F: factor deviationThe\ \ optimization problem for enhanced indexing:max_w d @ r - lamb * (v @ cov_b\ \ @ v + var_u @ d**2)s.t. w >= 0sum(w) == 1sum(|w - w0|) <= deltad >= -b_devd\ \ <= b_devv >= -f_devv <= f_dev" function: enhanced_indexing.EnhancedIndexingOptimizer - docstring: ' sub_fig_generator. it will return a generator, each row contains sub graphFIXME: Known limitation:- The last row will not be plotted automatically, please plot it outside the functionParameters----------sub_fs :the figure size of each subgraph in * subgraphscol_n :the number of subgraph in each row; It will generating a new graph after generating of subgraphs.row_n :the number of subgraph in each columnwspace :the width of the space for subgraphs in each rowhspace :the height of blank space for subgraphs in each columnYou can try 0.3 if you feel it is too crowdedReturns-------It will return graphs with the shape of each iter (it is squeezed).' function: utils.sub_fig_generator - docstring: " \nThis function `guesses` the rangebreaks required to remove gaps\ \ in datetime index.It basically calculates the difference between a `continuous`\ \ datetime index and index given.For more details on `rangebreaks` params in plotly,\ \ seehttps://plotly.com/python/reference/layout/xaxis/#layout-xaxis-rangebreaksParameters----------dt_index:\ \ pd.DatetimeIndexThe datetimes of the data.Returns-------the `rangebreaks` to\ \ be passed into plotly axis." function: utils.guess_plotly_rangebreaks - docstring: " \n:param df::param layout::param graph_kwargs::param name_dict::param\ \ kwargs:layout: dictgo.Layout parametersgraph_kwargs: dictGraph parameters, eg:\ \ go.Bar(**graph_kwargs)" function: 'graph.BaseGraph:' - docstring: " \n:return:" function: graph._init_data - docstring: " \n:param kwargs" function: graph._init_parameters - docstring: " \n:param graph_type::param kwargs::return:" function: graph.get_instance_with_graph_parameters - docstring: " \n:param figure_list::return:" function: graph.show_graph_in_notebook - docstring: " \n:return:" function: graph._get_layout - docstring: " \n:return:" function: graph._get_data - docstring: " \n:return:" function: graph.figure - docstring: null function: graph.ScatterGraph - docstring: null function: graph.BarGraph - docstring: null function: graph.DistplotGraph - docstring: " \n:return:" function: graph._get_data - docstring: null function: graph.HeatmapGraph - docstring: " \n:return:" function: graph._get_data - docstring: null function: graph.HistogramGraph - docstring: " \n:return:" function: graph._get_data - docstring: ' Create subplots same as df.plot(subplots=True) Simple package for `plotly.tools.subplots`' function: 'graph.SubplotsGraph:' - docstring: " \n:return:" function: graph._init_sub_graph_data - docstring: " \n:return:" function: graph._init_subplots_kwargs - docstring: " \n:return:" function: graph._init_figure - docstring: null function: 'base.FeaAnalyser:' - docstring: null function: base.calc_stat_values - docstring: null function: base.plot_single - docstring: null function: base.skip - docstring: " \nCombine the sub feature analysers and plot then in a single graph" function: ana.CombFeaAna - docstring: null function: ana.skip - docstring: ' The statistics of features are finished in the underlying analysers ' function: ana.calc_stat_values - docstring: null function: ana.plot_all - docstring: null function: ana.NumFeaAnalyser - docstring: null function: ana.skip - docstring: null function: ana.ValueCNT - docstring: null function: ana.calc_stat_values - docstring: null function: ana.plot_single - docstring: null function: ana.FeaDistAna - docstring: null function: ana.plot_single - docstring: null function: ana.FeaInfAna - docstring: null function: ana.calc_stat_values - docstring: null function: ana.skip - docstring: null function: ana.plot_single - docstring: null function: ana.FeaNanAna - docstring: null function: ana.calc_stat_values - docstring: null function: ana.skip - docstring: null function: ana.plot_single - docstring: null function: ana.FeaNanAnaRatio - docstring: null function: ana.calc_stat_values - docstring: null function: ana.skip - docstring: null function: ana.plot_single - docstring: ' Analysis the auto-correlation of features ' function: ana.FeaACAna - docstring: null function: ana.calc_stat_values - docstring: null function: ana.plot_single - docstring: null function: ana.FeaSkewTurt - docstring: null function: ana.calc_stat_values - docstring: null function: ana.plot_single - docstring: null function: ana.FeaMeanStd - docstring: null function: ana.calc_stat_values - docstring: null function: ana.plot_single - docstring: " \nMotivation:- display the values without further analysis" function: ana.RawFeaAna - docstring: null function: ana.calc_stat_values - docstring: " \n:param pred_label::param reverse::param N::return:" function: analysis_model_performance._group_return - docstring: " \n:param data::param dist::return:" function: analysis_model_performance._plot_qq - docstring: " \n:param pred_label: pd.DataFramemust contain one column of realized\ \ return with name `label` and one column of predicted score names `score`.:param\ \ methods: Sequence[Literal[\"IC\", \"Rank IC\"]]IC series to plot.IC is sectional\ \ pearson correlation between label and scoreRank IC is the spearman correlation\ \ between label and scoreFor the Monthly IC, IC histogram, IC Q-Q plot. Only\ \ the first type of IC will be plotted.:return:" function: analysis_model_performance._pred_ic - docstring: null function: analysis_model_performance._corr_series - docstring: null function: analysis_model_performance._pred_autocorr - docstring: null function: analysis_model_performance._pred_turnover - docstring: " \nif show_nature_day:date_index = pd.date_range(ic_df.index.min(),\ \ ic_df.index.max())ic_df = ic_df.reindex(date_index)ic_bar_figure = BarGraph(ic_df,layout=dict(title=\"\ Information Coefficient (IC)\",xaxis=dict(tickangle=45, rangebreaks=kwargs.get(\"\ rangebreaks\", guess_plotly_rangebreaks(ic_df.index))),),).figurereturn ic_bar_figure" function: analysis_model_performance.ic_figure - docstring: ' Parse position dict to position DataFrame :param position: position data:return: position DataFrame;.. code-block:: pythonposition_df = parse_position(positions)print(position_df.head())# status: 0-hold, -1-sell, 1-buyamount cash count price status weightinstrument datetimeSZ000547 2017-01-04 44.154290 211405.285654 1 205.189575 1 0.031255SZ300202 2017-01-04 60.638845 211405.285654 1 154.356506 1 0.032290SH600158 2017-01-04 46.531681 211405.285654 1 153.895142 1 0.024704SH600545 2017-01-04 197.173093 211405.285654 1 48.607037 1 0.033063SZ000930 2017-01-04 103.938300 211405.285654 1 80.759453 1 0.028958' function: parse_position.parse_position - docstring: ' Concat position with custom label :param position_df: position DataFrame:param label_data::return: concat result' function: parse_position._add_label_to_position - docstring: ' Concat position with bench :param position_df: position DataFrame:param bench: report normal data:return: concat result' function: parse_position._add_bench_to_position - docstring: ' calculate label rank :param df::return:' function: parse_position._calculate_label_rank - docstring: null function: parse_position._calculate_day_value - docstring: ' Concat position data with pred/report_normal :param position: position data:param report_normal: report normal, must be container ''bench'' column:param label_data::param calculate_label_rank::param start_date: start date:param end_date: end date:return: concat result,columns: [''amount'', ''cash'', ''count'', ''price'', ''status'', ''weight'', ''label'',''rank_ratio'', ''rank_label_mean'', ''excess_return'', ''score'', ''bench'']index: [''instrument'', ''date'']' function: parse_position.get_position_data - docstring: " \n:param position::param report_normal::param label_data::param\ \ start_date::param end_date::return:" function: cumulative_return._get_cum_return_data_with_position - docstring: ' Get average analysis figures :param position: position:param report_normal::param label_data::param start_date::param end_date::return:' function: cumulative_return._get_figure_with_position - docstring: ' Backtest buy, sell, and holding cumulative return graph Example:.. code-block:: pythonfrom qlib.data import Dfrom qlib.contrib.evaluate import risk_analysis, backtest, long_short_backtestfrom qlib.contrib.strategy import TopkDropoutStrategy# backtest parametersbparas = {}bparas[''limit_threshold''] = 0.095bparas[''account''] = 1000000000sparas = {}sparas[''topk''] = 50sparas[''n_drop''] = 5strategy = TopkDropoutStrategy(**sparas)report_normal_df, positions = backtest(pred_df, strategy, **bparas)pred_df_dates = pred_df.index.get_level_values(level=''datetime'')features_df = D.features(D.instruments(''csi500''), [''Ref($close, -1)/$close - 1''], pred_df_dates.min(), pred_df_dates.max())features_df.columns = [''label'']qcr.analysis_position.cumulative_return_graph(positions, report_normal_df, features_df)Graph desc:- Axis X: Trading day.- Axis Y:- Above axis Y: `(((Ref($close, -1)/$close - 1) * weight).sum() / weight.sum()).cumsum()`.- Below axis Y: Daily weight sum.- In the **sell** graph, `y < 0` stands for profit; in other cases, `y > 0` stands for profit.- In the **buy_minus_sell** graph, the **y** value of the **weight** graph at the bottom is `buy_weight + sell_weight`.- In each graph, the **red line** in the histogram on the right represents the average.:param position: position data:param report_normal:.. code-block:: pythonreturn cost bench turnoverdate2017-01-04 0.003421 0.000864 0.011693 0.5763252017-01-05 0.000508 0.000447 0.000721 0.2278822017-01-06 -0.003321 0.000212 -0.004322 0.1027652017-01-09 0.006753 0.000212 0.006874 0.1058642017-01-10 -0.000416 0.000440 -0.003350 0.208396:param label_data: `D.features` result; index is `pd.MultiIndex`, index name is [`instrument`, `datetime`]; columns names is [`label`].**The label T is the change from T to T+1**, it is recommended to use ``close``, example: `D.features(D.instruments(''csi500''), [''Ref($close, -1)/$close-1''])`.. code-block:: pythonlabelinstrument datetimeSH600004 2017-12-11 -0.0135022017-12-12 -0.0723672017-12-13 -0.0686052017-12-14 0.0124402017-12-15 -0.102778:param show_notebook: True or False. If True, show graph in notebook, else return figures:param start_date: start date:param end_date: end date:return:' function: cumulative_return.cumulative_return_graph - docstring: ' Get average analysis figures :param position: position:param label_data::param start_date::param end_date::return:' function: rank_label._get_figure_with_position - docstring: ' Ranking percentage of stocks buy, sell, and holding on the trading day. Average rank-ratio(similar to **sell_df[''label''].rank(ascending=False) / len(sell_df)**) of daily tradingExample:.. code-block:: pythonfrom qlib.data import Dfrom qlib.contrib.evaluate import backtestfrom qlib.contrib.strategy import TopkDropoutStrategy# backtest parametersbparas = {}bparas[''limit_threshold''] = 0.095bparas[''account''] = 1000000000sparas = {}sparas[''topk''] = 50sparas[''n_drop''] = 230strategy = TopkDropoutStrategy(**sparas)_, positions = backtest(pred_df, strategy, **bparas)pred_df_dates = pred_df.index.get_level_values(level=''datetime'')features_df = D.features(D.instruments(''csi500''), [''Ref($close, -1)/$close-1''], pred_df_dates.min(), pred_df_dates.max())features_df.columns = [''label'']qcr.analysis_position.rank_label_graph(positions, features_df, pred_df_dates.min(), pred_df_dates.max()):param position: position data; **qlib.backtest.backtest** result.:param label_data: **D.features** result; index is **pd.MultiIndex**, index name is **[instrument, datetime]**; columns names is **[label]**.**The label T is the change from T to T+1**, it is recommended to use ``close``, example: `D.features(D.instruments(''csi500''), [''Ref($close, -1)/$close-1''])`... code-block:: pythonlabelinstrument datetimeSH600004 2017-12-11 -0.0135022017-12-12 -0.0723672017-12-13 -0.0686052017-12-14 0.0124402017-12-15 -0.102778:param start_date: start date:param end_date: end_date:param show_notebook: **True** or **False**. If True, show graph in notebook, else return figures.:return:' function: rank_label.rank_label_graph - docstring: " \n:param df::param is_ex::return:" function: report._calculate_maximum - docstring: " \nCalculate mdd:param series::return:" function: report._calculate_mdd - docstring: " \n:param df::return:" function: report._calculate_report_data - docstring: " \n:param df::return:" function: report._report_figure - docstring: ' display backtest report Example:.. code-block:: pythonimport qlibimport pandas as pdfrom qlib.utils.time import Freqfrom qlib.utils import flatten_dictfrom qlib.backtest import backtest, executorfrom qlib.contrib.evaluate import risk_analysisfrom qlib.contrib.strategy import TopkDropoutStrategy# init qlibqlib.init(provider_uri=)CSI300_BENCH = "SH000300"FREQ = "day"STRATEGY_CONFIG = {"topk": 50,"n_drop": 5,# pred_score, pd.Series"signal": pred_score,}EXECUTOR_CONFIG = {"time_per_step": "day","generate_portfolio_metrics": True,}backtest_config = {"start_time": "2017-01-01","end_time": "2020-08-01","account": 100000000,"benchmark": CSI300_BENCH,"exchange_kwargs": {"freq": FREQ,"limit_threshold": 0.095,"deal_price": "close","open_cost": 0.0005,"close_cost": 0.0015,"min_cost": 5,},}# strategy objectstrategy_obj = TopkDropoutStrategy(**STRATEGY_CONFIG)# executor objectexecutor_obj = executor.SimulatorExecutor(**EXECUTOR_CONFIG)# backtestportfolio_metric_dict, indicator_dict = backtest(executor=executor_obj, strategy=strategy_obj, **backtest_config)analysis_freq = "{0}{1}".format(*Freq.parse(FREQ))# backtest inforeport_normal_df, positions_normal = portfolio_metric_dict.get(analysis_freq)qcr.analysis_position.report_graph(report_normal_df):param report_df: **df.index.name** must be **date**, **df.columns** must contain **return**, **turnover**, **cost**, **bench**... code-block:: pythonreturn cost bench turnoverdate2017-01-04 0.003421 0.000864 0.011693 0.5763252017-01-05 0.000508 0.000447 0.000721 0.2278822017-01-06 -0.003321 0.000212 -0.004322 0.1027652017-01-09 0.006753 0.000212 0.006874 0.1058642017-01-10 -0.000416 0.000440 -0.003350 0.208396:param show_notebook: whether to display graphics in notebook, the default is **True**.:return: if show_notebook is True, display in notebook; else return **plotly.graph_objs.Figure** list.' function: report.report_graph - docstring: ' Get risk analysis data with report :param report_normal_df: report data:param report_long_short_df: report data:param date: date string:return:' function: risk_analysis._get_risk_analysis_data_with_report - docstring: ' risk_df to standard :param risk_df: risk data:return:' function: risk_analysis._get_all_risk_analysis - docstring: ' Get monthly analysis data :param report_normal_df:# :param report_long_short_df::return:' function: risk_analysis._get_monthly_risk_analysis_with_report - docstring: " \n:param monthly_df::param feature::return:" function: risk_analysis._get_monthly_analysis_with_feature - docstring: ' Get analysis graph figure :param analysis_df::return:' function: risk_analysis._get_risk_analysis_figure - docstring: ' Get analysis monthly graph figure :param report_normal_df::param report_long_short_df::return:' function: risk_analysis._get_monthly_risk_analysis_figure - docstring: ' Generate analysis graph and monthly analysis Example:.. code-block:: pythonimport qlibimport pandas as pdfrom qlib.utils.time import Freqfrom qlib.utils import flatten_dictfrom qlib.backtest import backtest, executorfrom qlib.contrib.evaluate import risk_analysisfrom qlib.contrib.strategy import TopkDropoutStrategy# init qlibqlib.init(provider_uri=)CSI300_BENCH = "SH000300"FREQ = "day"STRATEGY_CONFIG = {"topk": 50,"n_drop": 5,# pred_score, pd.Series"signal": pred_score,}EXECUTOR_CONFIG = {"time_per_step": "day","generate_portfolio_metrics": True,}backtest_config = {"start_time": "2017-01-01","end_time": "2020-08-01","account": 100000000,"benchmark": CSI300_BENCH,"exchange_kwargs": {"freq": FREQ,"limit_threshold": 0.095,"deal_price": "close","open_cost": 0.0005,"close_cost": 0.0015,"min_cost": 5,},}# strategy objectstrategy_obj = TopkDropoutStrategy(**STRATEGY_CONFIG)# executor objectexecutor_obj = executor.SimulatorExecutor(**EXECUTOR_CONFIG)# backtestportfolio_metric_dict, indicator_dict = backtest(executor=executor_obj, strategy=strategy_obj, **backtest_config)analysis_freq = "{0}{1}".format(*Freq.parse(FREQ))# backtest inforeport_normal_df, positions_normal = portfolio_metric_dict.get(analysis_freq)analysis = dict()analysis["excess_return_without_cost"] = risk_analysis(report_normal_df["return"] - report_normal_df["bench"], freq=analysis_freq)analysis["excess_return_with_cost"] = risk_analysis(report_normal_df["return"] - report_normal_df["bench"] - report_normal_df["cost"], freq=analysis_freq)analysis_df = pd.concat(analysis) # type: pd.DataFrameanalysis_position.risk_analysis_graph(analysis_df, report_normal_df):param analysis_df: analysis data, index is **pd.MultiIndex**; columns names is **[risk]**... code-block:: pythonriskexcess_return_without_cost mean 0.000692std 0.005374annualized_return 0.174495information_ratio 2.045576max_drawdown -0.079103excess_return_with_cost mean 0.000499std 0.005372annualized_return 0.125625information_ratio 1.473152max_drawdown -0.088263:param report_normal_df: **df.index.name** must be **date**, df.columns must contain **return**, **turnover**, **cost**, **bench**... code-block:: pythonreturn cost bench turnoverdate2017-01-04 0.003421 0.000864 0.011693 0.5763252017-01-05 0.000508 0.000447 0.000721 0.2278822017-01-06 -0.003321 0.000212 -0.004322 0.1027652017-01-09 0.006753 0.000212 0.006874 0.1058642017-01-10 -0.000416 0.000440 -0.003350 0.208396:param report_long_short_df: **df.index.name** must be **date**, df.columns contain **long**, **short**, **long_short**... code-block:: pythonlong short long_shortdate2017-01-04 -0.001360 0.001394 0.0000342017-01-05 0.002456 0.000058 0.0025142017-01-06 0.000120 0.002739 0.0028592017-01-09 0.001436 0.001838 0.0032732017-01-10 0.000824 -0.001944 -0.001120:param show_notebook: Whether to display graphics in a notebook, default **True**.If True, show graph in notebookIf False, return graph figure:return:' function: risk_analysis.risk_analysis_graph - docstring: " \n:param pred_label::return:" function: score_ic._get_score_ic - docstring: ' score IC Example:.. code-block:: pythonfrom qlib.data import Dfrom qlib.contrib.report import analysis_positionpred_df_dates = pred_df.index.get_level_values(level=''datetime'')features_df = D.features(D.instruments(''csi500''), [''Ref($close, -2)/Ref($close, -1)-1''], pred_df_dates.min(), pred_df_dates.max())features_df.columns = [''label'']pred_label = pd.concat([features_df, pred], axis=1, sort=True).reindex(features_df.index)analysis_position.score_ic_graph(pred_label):param pred_label: index is **pd.MultiIndex**, index name is **[instrument, datetime]**; columns names is **[score, label]**... code-block:: pythoninstrument datetime score labelSH600004 2017-12-11 -0.013502 -0.0135022017-12-12 -0.072367 -0.0723672017-12-13 -0.068605 -0.0686052017-12-14 0.012440 0.0124402017-12-15 -0.102778 -0.102778:param show_notebook: whether to display graphics in notebook, the default is **True**.:return: if show_notebook is True, display in notebook; else return **plotly.graph_objs.Figure** list.' function: score_ic.score_ic_graph - docstring: " \ncalculate the precision for long and short operation:param pred/label:\ \ index is **pd.MultiIndex**, index name is **[datetime, instruments]**; columns\ \ names is **[score]**... code-block:: pythonscoredatetime instrument2020-12-01\ \ 09:30:00 SH600068 0.553634SH600195 0.550017SH600276 0.540321SH600584\ \ 0.517297SH600715 0.544674label :labeldate_col :date_colReturns-------(pd.Series,\ \ pd.Series)long precision and short precision in time level" function: alpha.calc_long_short_prec - docstring: null function: alpha.N - docstring: " \ncalculate long-short returnNote:`label` must be raw stock returns.Parameters----------pred\ \ : pd.Seriesstock predictionslabel : pd.Seriesstock returnsdate_col : strdatetime\ \ index namequantile : floatlong-short quantileReturns----------long_short_r :\ \ pd.Seriesdaily long-short returnslong_avg_r : pd.Seriesdaily long-average returns" function: alpha.calc_long_short_return - docstring: null function: alpha.N - docstring: ' pred_autocorr. Limitation:- If the datetime is not sequential densely, the correlation will be calulated based on adjacent dates. (some users may expected NaN):param pred: pd.Series with following formatinstrument datetimeSH600000 2016-01-04 -0.0004032016-01-05 -0.0007532016-01-06 -0.0218012016-01-07 -0.0652302016-01-08 -0.062465:type pred: pd.Series:param lag:' function: alpha.pred_autocorr - docstring: " \ncalculate auto correlation for pred_dictParameters----------pred_dict\ \ : dictA dict like {: }kwargs :all these arguments\ \ will be passed into pred_autocorr" function: alpha.pred_autocorr_all - docstring: ' calc_ic. Parameters----------pred :predlabel :labeldate_col :date_colReturns-------(pd.Series, pd.Series)ic and rank ic' function: alpha.calc_ic - docstring: ' calc_all_ic. Parameters----------pred_dict_all :A dict like {: }label:A pd.Series of label valuesReturns-------{''Q2+IND_z'': {''ic'': 2016-01-04 -0.057407...2020-05-28 0.1834702020-05-29 0.171393''ric'': 2016-01-04 -0.040888...2020-05-28 0.2366652020-05-29 0.183886}...}' function: alpha.calc_all_ic - docstring: null function: handler.check_transform_proc - docstring: null function: handler.Alpha360 - docstring: null function: handler.get_label_config - docstring: null function: handler.get_feature_config - docstring: null function: handler.Alpha360vwap - docstring: null function: handler.get_label_config - docstring: null function: handler.Alpha158 - docstring: null function: handler.get_feature_config - docstring: null function: handler.get_label_config - docstring: ' create factors from config config = {''kbar'': {}, # whether to use some hard-code kbar features''price'': { # whether to use raw price features''windows'': [0, 1, 2, 3, 4], # use price at n days ago''feature'': [''OPEN'', ''HIGH'', ''LOW''] # which price field to use},''volume'': { # whether to use raw volume features''windows'': [0, 1, 2, 3, 4], # use volume at n days ago},''rolling'': { # whether to use rolling operator based features''windows'': [5, 10, 20, 30, 60], # rolling windows size''include'': [''ROC'', ''MA'', ''STD''], # rolling operator to use#if include is None we will use default operators''exclude'': [''RANK''], # rolling operator not to use}}' function: handler.parse_config_to_fields - docstring: null function: handler.use - docstring: null function: handler.Alpha158vwap - docstring: null function: data.ArcticFeatureProvider - docstring: null function: dataset._to_tensor - docstring: " \ncreate time series slices from pandas indexArgs:index (pd.MultiIndex):\ \ pandas multiindex with orderseq_len (int): sequence length" function: dataset._create_ts_slices - docstring: ' get date parse function This method is used to parse date arguments as target type.Example:get_date_parse_fn(''20120101'')(''2017-01-01'') => ''20170101''get_date_parse_fn(20120101)(''2017-01-01'') => 20170101' function: dataset._get_date_parse_fn - docstring: null function: dataset._fn - docstring: null function: dataset._fn - docstring: null function: dataset._fn - docstring: ' padding 2d