liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-15 16:56:54 +08:00
Code Activity

fixed a problem with multi index caused by the default value of groupkey (#1917)

Browse Source 
* fixed a problem with multi index caused by the default value of groupkey

* modify group_key default value

* limit pandas verion

* format with black

* fix docs error

* fix docs error

* fixed bugs caused by pandas upgrade

* remove needless code

* reformat with black

* limit version & add docs
This commit is contained in:
Linlang
2025-05-13 16:02:49 +08:00
committed by GitHub
parent df557d29d5
commit fbba768006
43 changed files with 153 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
qlib/contrib/data/dataset.py
View File

@@ -32,7 +32,7 @@ def _create_ts_slices(index, seq_len):
assert index.is_monotonic_increasing, "index should be sorted"
# number of dates for each instrument
sample_count_by_insts = index.to_series().groupby(level=0).size().values
sample_count_by_insts = index.to_series().groupby(level=0, group_keys=False).size().values
# start index for each instrument
start_index_of_insts = np.roll(np.cumsum(sample_count_by_insts), 1)

34
qlib/contrib/data/processor.py
View File

@@ -55,14 +55,18 @@ class ConfigSectionProcessor(Processor):
# Label
cols = df_focus.columns[df_focus.columns.str.contains("^LABEL")]
df_focus[cols] = df_focus[cols].groupby(level="datetime").apply(_label_norm)
df_focus[cols] = df_focus[cols].groupby(level="datetime", group_keys=False).apply(_label_norm)
# Features
cols = df_focus.columns[df_focus.columns.str.contains("^KLEN|^KLOW|^KUP")]
df_focus[cols] = df_focus[cols].apply(lambda x: x**0.25).groupby(level="datetime").apply(_feature_norm)
df_focus[cols] = (
df_focus[cols].apply(lambda x: x**0.25).groupby(level="datetime", group_keys=False).apply(_feature_norm)
)
cols = df_focus.columns[df_focus.columns.str.contains("^KLOW2|^KUP2")]
df_focus[cols] = df_focus[cols].apply(lambda x: x**0.5).groupby(level="datetime").apply(_feature_norm)
df_focus[cols] = (
df_focus[cols].apply(lambda x: x**0.5).groupby(level="datetime", group_keys=False).apply(_feature_norm)
)
_cols = [
"KMID",
@@ -88,25 +92,35 @@ class ConfigSectionProcessor(Processor):
]
pat = "|".join(["^" + x for x in _cols])
cols = df_focus.columns[df_focus.columns.str.contains(pat) & (~df_focus.columns.isin(["HIGH0", "LOW0"]))]
df_focus[cols] = df_focus[cols].groupby(level="datetime").apply(_feature_norm)
df_focus[cols] = df_focus[cols].groupby(level="datetime", group_keys=False).apply(_feature_norm)
cols = df_focus.columns[df_focus.columns.str.contains("^STD|^VOLUME|^VMA|^VSTD")]
df_focus[cols] = df_focus[cols].apply(np.log).groupby(level="datetime").apply(_feature_norm)
df_focus[cols] = df_focus[cols].apply(np.log).groupby(level="datetime", group_keys=False).apply(_feature_norm)
cols = df_focus.columns[df_focus.columns.str.contains("^RSQR")]
df_focus[cols] = df_focus[cols].fillna(0).groupby(level="datetime").apply(_feature_norm)
df_focus[cols] = df_focus[cols].fillna(0).groupby(level="datetime", group_keys=False).apply(_feature_norm)
cols = df_focus.columns[df_focus.columns.str.contains("^MAX|^HIGH0")]
df_focus[cols] = df_focus[cols].apply(lambda x: (x - 1) ** 0.5).groupby(level="datetime").apply(_feature_norm)
df_focus[cols] = (
df_focus[cols]
.apply(lambda x: (x - 1) ** 0.5)
.groupby(level="datetime", group_keys=False)
.apply(_feature_norm)
)
cols = df_focus.columns[df_focus.columns.str.contains("^MIN|^LOW0")]
df_focus[cols] = df_focus[cols].apply(lambda x: (1 - x) ** 0.5).groupby(level="datetime").apply(_feature_norm)
df_focus[cols] = (
df_focus[cols]
.apply(lambda x: (1 - x) ** 0.5)
.groupby(level="datetime", group_keys=False)
.apply(_feature_norm)
)
cols = df_focus.columns[df_focus.columns.str.contains("^CORR|^CORD")]
df_focus[cols] = df_focus[cols].apply(np.exp).groupby(level="datetime").apply(_feature_norm)
df_focus[cols] = df_focus[cols].apply(np.exp).groupby(level="datetime", group_keys=False).apply(_feature_norm)
cols = df_focus.columns[df_focus.columns.str.contains("^WVMA")]
df_focus[cols] = df_focus[cols].apply(np.log1p).groupby(level="datetime").apply(_feature_norm)
df_focus[cols] = df_focus[cols].apply(np.log1p).groupby(level="datetime", group_keys=False).apply(_feature_norm)
df[selected_cols] = df_focus.values

22
qlib/contrib/eva/alpha.py
View File

@@ -39,7 +39,7 @@ def calc_long_short_prec(
long precision and short precision in time level
"""
if is_alpha:
label = label - label.mean(level=date_col)
label = label - label.groupby(level=date_col, group_keys=False).mean()
if int(1 / quantile) >= len(label.index.get_level_values(1).unique()):
raise ValueError("Need more instruments to calculate precision")
@@ -47,23 +47,25 @@ def calc_long_short_prec(
if dropna:
df.dropna(inplace=True)
group = df.groupby(level=date_col)
group = df.groupby(level=date_col, group_keys=False)
def N(x):
return int(len(x) * quantile)
# find the top/low quantile of prediction and treat them as long and short target
long = group.apply(lambda x: x.nlargest(N(x), columns="pred").label).reset_index(level=0, drop=True)
short = group.apply(lambda x: x.nsmallest(N(x), columns="pred").label).reset_index(level=0, drop=True)
long = group.apply(lambda x: x.nlargest(N(x), columns="pred").label)
short = group.apply(lambda x: x.nsmallest(N(x), columns="pred").label)
groupll = long.groupby(date_col)
groupll = long.groupby(date_col, group_keys=False)
l_dom = groupll.apply(lambda x: x > 0)
l_c = groupll.count()
groups = short.groupby(date_col)
groups = short.groupby(date_col, group_keys=False)
s_dom = groups.apply(lambda x: x < 0)
s_c = groups.count()
return (l_dom.groupby(date_col).sum() / l_c), (s_dom.groupby(date_col).sum() / s_c)
return (l_dom.groupby(date_col, group_keys=False).sum() / l_c), (
s_dom.groupby(date_col, group_keys=False).sum() / s_c
)
def calc_long_short_return(
@@ -100,7 +102,7 @@ def calc_long_short_return(
df = pd.DataFrame({"pred": pred, "label": label})
if dropna:
df.dropna(inplace=True)
group = df.groupby(level=date_col)
group = df.groupby(level=date_col, group_keys=False)
def N(x):
return int(len(x) * quantile)
@@ -173,8 +175,8 @@ def calc_ic(pred: pd.Series, label: pd.Series, date_col="datetime", dropna=False
ic and rank ic
"""
df = pd.DataFrame({"pred": pred, "label": label})
ic = df.groupby(date_col).apply(lambda df: df["pred"].corr(df["label"]))
ric = df.groupby(date_col).apply(lambda df: df["pred"].corr(df["label"], method="spearman"))
ic = df.groupby(date_col, group_keys=False).apply(lambda df: df["pred"].corr(df["label"]))
ric = df.groupby(date_col, group_keys=False).apply(lambda df: df["pred"].corr(df["label"], method="spearman"))
if dropna:
return ic.dropna(), ric.dropna()
else:

4
qlib/contrib/meta/data_selection/dataset.py
View File

@@ -106,7 +106,7 @@ class InternalData:
def _calc_perf(self, pred, label):
df = pd.DataFrame({"pred": pred, "label": label})
df = df.groupby("datetime").corr(method="spearman")
df = df.groupby("datetime", group_keys=False).corr(method="spearman")
corr = df.loc(axis=0)[:, "pred"]["label"].droplevel(axis=0, level=-1)
return corr
@@ -161,7 +161,7 @@ class MetaTaskDS(MetaTask):
raise ValueError(f"Most of samples are dropped. Please check this task: {task}")
assert (
d_test.groupby("datetime").size().shape[0] >= 5
d_test.groupby("datetime", group_keys=False).size().shape[0] >= 5
), "In this segment, this trading dates is less than 5, you'd better check the data."
sample_time_belong = np.zeros((d_train.shape[0], time_perf.shape[1]))

6
qlib/contrib/meta/data_selection/model.py
View File

@@ -125,7 +125,11 @@ class MetaModelDS(MetaTaskModel):
loss_l.setdefault(phase, []).append(running_loss)
pred_y_all = pd.concat(pred_y_all)
ic = pred_y_all.groupby("datetime").apply(lambda df: df["pred"].corr(df["label"], method="spearman")).mean()
ic = (
pred_y_all.groupby("datetime", group_keys=False)
.apply(lambda df: df["pred"].corr(df["label"], method="spearman"))
.mean()
)
R.log_metrics(**{f"loss/{phase}": running_loss, "step": epoch})
R.log_metrics(**{f"ic/{phase}": ic, "step": epoch})

2
qlib/contrib/model/double_ensemble.py
View File

@@ -166,7 +166,7 @@ class DEnsembleModel(Model, FeatureInt):
# calculate weights
h["bins"] = pd.cut(h["h_value"], self.bins_sr)
h_avg = h.groupby("bins")["h_value"].mean()
h_avg = h.groupby("bins", group_keys=False, observed=False)["h_value"].mean()
weights = pd.Series(np.zeros(N, dtype=float))
for b in h_avg.index:
weights[h["bins"] == b] = 1.0 / (self.decay**k_th * h_avg[b] + 0.1)

10
qlib/contrib/model/highfreq_gdbt_model.py
View File

@@ -90,8 +90,14 @@ class HFLGBModel(ModelFT, LightGBMFInt):
if y_train.values.ndim == 2 and y_train.values.shape[1] == 1:
l_name = df_train["label"].columns[0]
# Convert label into alpha
df_train["label"][l_name] = df_train["label"][l_name] - df_train["label"][l_name].mean(level=0)
df_valid["label"][l_name] = df_valid["label"][l_name] - df_valid["label"][l_name].mean(level=0)
df_train.loc[:, ("label", l_name)] = (
df_train.loc[:, ("label", l_name)]
- df_train.loc[:, ("label", l_name)].groupby(level=0, group_keys=False).mean()
)
df_valid.loc[:, ("label", l_name)] = (
df_valid.loc[:, ("label", l_name)]
- df_valid.loc[:, ("label", l_name)].groupby(level=0, group_keys=False).mean()
)
def mapping_fn(x):
return 0 if x < 0 else 1

6
qlib/contrib/model/pytorch_adarnn.py
View File

@@ -214,8 +214,10 @@ class ADARNN(Model):
def calc_all_metrics(pred):
"""pred is a pandas dataframe that has two attributes: score (pred) and label (real)"""
res = {}
ic = pred.groupby(level="datetime").apply(lambda x: x.label.corr(x.score))
rank_ic = pred.groupby(level="datetime").apply(lambda x: x.label.corr(x.score, method="spearman"))
ic = pred.groupby(level="datetime", group_keys=False).apply(lambda x: x.label.corr(x.score))
rank_ic = pred.groupby(level="datetime", group_keys=False).apply(
lambda x: x.label.corr(x.score, method="spearman")
)
res["ic"] = ic.mean()
res["icir"] = ic.mean() / ic.std()
res["ric"] = rank_ic.mean()

6
qlib/contrib/model/pytorch_add.py
View File

@@ -226,7 +226,7 @@ class ADD(Model):
def get_daily_inter(self, df, shuffle=False):
# organize the train data into daily batches
daily_count = df.groupby(level=0).size().values
daily_count = df.groupby(level=0, group_keys=False).size().values
daily_index = np.roll(np.cumsum(daily_count), 1)
daily_index[0] = 0
if shuffle:
@@ -349,7 +349,7 @@ class ADD(Model):
return best_score
def gen_market_label(self, df, raw_label):
market_label = raw_label.groupby("datetime").mean().squeeze()
market_label = raw_label.groupby("datetime", group_keys=False).mean().squeeze()
bins = [-np.inf, self.lo, self.hi, np.inf]
market_label = pd.cut(market_label, bins, labels=False)
market_label.name = ("market_return", "market_return")
@@ -357,7 +357,7 @@ class ADD(Model):
return df
def fit_thresh(self, train_label):
market_label = train_label.groupby("datetime").mean().squeeze()
market_label = train_label.groupby("datetime", group_keys=False).mean().squeeze()
self.lo, self.hi = market_label.quantile([1 / 3, 2 / 3])
def fit(

2
qlib/contrib/model/pytorch_gats.py
View File

@@ -163,7 +163,7 @@ class GATs(Model):
def get_daily_inter(self, df, shuffle=False):
# organize the train data into daily batches
daily_count = df.groupby(level=0).size().values
daily_count = df.groupby(level=0, group_keys=False).size().values
daily_index = np.roll(np.cumsum(daily_count), 1)
daily_index[0] = 0
if shuffle:

6
qlib/contrib/model/pytorch_gats_ts.py
View File

@@ -27,7 +27,9 @@ class DailyBatchSampler(Sampler):
def __init__(self, data_source):
self.data_source = data_source
# calculate number of samples in each batch
self.daily_count = pd.Series(index=self.data_source.get_index()).groupby("datetime").size().values
self.daily_count = (
pd.Series(index=self.data_source.get_index()).groupby("datetime", group_keys=False).size().values
)
self.daily_index = np.roll(np.cumsum(self.daily_count), 1) # calculate begin index of each batch
self.daily_index[0] = 0
@@ -181,7 +183,7 @@ class GATs(Model):
def get_daily_inter(self, df, shuffle=False):
# organize the train data into daily batches
daily_count = df.groupby(level=0).size().values
daily_count = df.groupby(level=0, group_keys=False).size().values
daily_index = np.roll(np.cumsum(daily_count), 1)
daily_index[0] = 0
if shuffle:

2
qlib/contrib/model/pytorch_hist.py
View File

@@ -177,7 +177,7 @@ class HIST(Model):
def get_daily_inter(self, df, shuffle=False):
# organize the train data into daily batches
daily_count = df.groupby(level=0).size().values
daily_count = df.groupby(level=0, group_keys=False).size().values
daily_index = np.roll(np.cumsum(daily_count), 1)
daily_index[0] = 0
if shuffle:

2
qlib/contrib/model/pytorch_igmtf.py
View File

@@ -170,7 +170,7 @@ class IGMTF(Model):
def get_daily_inter(self, df, shuffle=False):
# organize the train data into daily batches
daily_count = df.groupby(level=0).size().values
daily_count = df.groupby(level=0, group_keys=False).size().values
daily_index = np.roll(np.cumsum(daily_count), 1)
daily_index[0] = 0
if shuffle:

2
qlib/contrib/model/pytorch_krnn.py
View File

@@ -368,7 +368,7 @@ class KRNN(Model):
def get_daily_inter(self, df, shuffle=False):
# organize the train data into daily batches
daily_count = df.groupby(level=0).size().values
daily_count = df.groupby(level=0, group_keys=False).size().values
daily_index = np.roll(np.cumsum(daily_count), 1)
daily_index[0] = 0
if shuffle:

4
qlib/contrib/ops/high_freq.py
View File

@@ -96,7 +96,7 @@ class DayCumsum(ElemOperator):
def _load_internal(self, instrument, start_index, end_index, freq):
_calendar = get_calendar_day(freq=freq)
series = self.feature.load(instrument, start_index, end_index, freq)
return series.groupby(_calendar[series.index]).transform(self.period_cusum)
return series.groupby(_calendar[series.index], group_keys=False).transform(self.period_cusum)
class DayLast(ElemOperator):
@@ -116,7 +116,7 @@ class DayLast(ElemOperator):
def _load_internal(self, instrument, start_index, end_index, freq):
_calendar = get_calendar_day(freq=freq)
series = self.feature.load(instrument, start_index, end_index, freq)
return series.groupby(_calendar[series.index]).transform("last")
return series.groupby(_calendar[series.index], group_keys=False).transform("last")
class FFillNan(ElemOperator):

24
qlib/contrib/report/analysis_model/analysis_model_performance.py
View File

@@ -38,7 +38,7 @@ def _group_return(pred_label: pd.DataFrame = None, reverse: bool = False, N: int
t_df = pd.DataFrame(
{
"Group%d"
% (i + 1): pred_label_drop.groupby(level="datetime")["label"].apply(
% (i + 1): pred_label_drop.groupby(level="datetime", group_keys=False)["label"].apply(
lambda x: x[len(x) // N * i : len(x) // N * (i + 1)].mean() # pylint: disable=W0640
)
for i in range(N)
@@ -50,7 +50,7 @@ def _group_return(pred_label: pd.DataFrame = None, reverse: bool = False, N: int
t_df["long-short"] = t_df["Group1"] - t_df["Group%d" % N]
# Long-Average
t_df["long-average"] = t_df["Group1"] - pred_label.groupby(level="datetime")["label"].mean()
t_df["long-average"] = t_df["Group1"] - pred_label.groupby(level="datetime", group_keys=False)["label"].mean()
t_df = t_df.dropna(how="all") # for days which does not contain label
# Cumulative Return By Group
@@ -137,7 +137,9 @@ def _pred_ic(
ic_df = pd.concat(
[
pred_label.groupby(level="datetime").apply(partial(_corr_series, method=_methods_mapping[m])).rename(m)
pred_label.groupby(level="datetime", group_keys=False)
.apply(partial(_corr_series, method=_methods_mapping[m]))
.rename(m)
for m in methods
],
axis=1,
@@ -145,7 +147,7 @@ def _pred_ic(
_ic = ic_df.iloc(axis=1)[0]
_index = _ic.index.get_level_values(0).astype("str").str.replace("-", "").str.slice(0, 6)
_monthly_ic = _ic.groupby(_index).mean()
_monthly_ic = _ic.groupby(_index, group_keys=False).mean()
_monthly_ic.index = pd.MultiIndex.from_arrays(
[_monthly_ic.index.str.slice(0, 4), _monthly_ic.index.str.slice(4, 6)],
names=["year", "month"],
@@ -220,8 +222,10 @@ def _pred_ic(
def _pred_autocorr(pred_label: pd.DataFrame, lag=1, **kwargs) -> tuple:
pred = pred_label.copy()
pred["score_last"] = pred.groupby(level="instrument")["score"].shift(lag)
ac = pred.groupby(level="datetime").apply(lambda x: x["score"].rank(pct=True).corr(x["score_last"].rank(pct=True)))
pred["score_last"] = pred.groupby(level="instrument", group_keys=False)["score"].shift(lag)
ac = pred.groupby(level="datetime", group_keys=False).apply(
lambda x: x["score"].rank(pct=True).corr(x["score_last"].rank(pct=True))
)
_df = ac.to_frame("value")
ac_figure = ScatterGraph(
_df,
@@ -235,13 +239,13 @@ def _pred_autocorr(pred_label: pd.DataFrame, lag=1, **kwargs) -> tuple:
def _pred_turnover(pred_label: pd.DataFrame, N=5, lag=1, **kwargs) -> tuple:
pred = pred_label.copy()
pred["score_last"] = pred.groupby(level="instrument")["score"].shift(lag)
top = pred.groupby(level="datetime").apply(
pred["score_last"] = pred.groupby(level="instrument", group_keys=False)["score"].shift(lag)
top = pred.groupby(level="datetime", group_keys=False).apply(
lambda x: 1
- x.nlargest(len(x) // N, columns="score").index.isin(x.nlargest(len(x) // N, columns="score_last").index).sum()
/ (len(x) // N)
)
bottom = pred.groupby(level="datetime").apply(
bottom = pred.groupby(level="datetime", group_keys=False).apply(
lambda x: 1
- x.nsmallest(len(x) // N, columns="score")
.index.isin(x.nsmallest(len(x) // N, columns="score_last").index)
@@ -313,7 +317,7 @@ def model_performance_graph(
2017-12-15 -0.102778 -0.102778
:param lag: `pred.groupby(level='instrument')['score'].shift(lag)`. It will be only used in the auto-correlation computing.
:param lag: `pred.groupby(level='instrument', group_keys=False)['score'].shift(lag)`. It will be only used in the auto-correlation computing.
:param N: group number, default 5.
:param reverse: if `True`, `pred['score'] *= -1`.
:param rank: if **True**, calculate rank ic.

2
qlib/contrib/report/analysis_position/cumulative_return.py
View File

@@ -38,7 +38,7 @@ def _get_cum_return_data_with_position(
_cumulative_return_df["label"] = _cumulative_return_df["label"] - _cumulative_return_df["bench"]
_cumulative_return_df = _cumulative_return_df.dropna()
df_gp = _cumulative_return_df.groupby(level="datetime")
df_gp = _cumulative_return_df.groupby(level="datetime", group_keys=False)
result_list = []
for gp in df_gp:
date = gp[0]

2
qlib/contrib/report/analysis_position/parse_position.py
View File

@@ -132,7 +132,7 @@ def _calculate_label_rank(df: pd.DataFrame) -> pd.DataFrame:
g_df["excess_return"] = g_df[_label_name] - g_df[_label_name].mean()
return g_df
return df.groupby(level="datetime").apply(_calculate_day_value)
return df.groupby(level="datetime", group_keys=False).apply(_calculate_day_value)
def get_position_data(

2
qlib/contrib/report/analysis_position/rank_label.py
View File

@@ -31,7 +31,7 @@ def _get_figure_with_position(
)
res_dict = dict()
_pos_gp = _position_df.groupby(level=1)
_pos_gp = _position_df.groupby(level=1, group_keys=False)
for _item in _pos_gp:
_date = _item[0]
_day_df = _item[1]

8
qlib/contrib/report/analysis_position/risk_analysis.py
View File

@@ -63,9 +63,11 @@ def _get_monthly_risk_analysis_with_report(report_normal_df: pd.DataFrame) -> pd
"""
# Group by month
report_normal_gp = report_normal_df.groupby([report_normal_df.index.year, report_normal_df.index.month])
report_normal_gp = report_normal_df.groupby(
[report_normal_df.index.year, report_normal_df.index.month], group_keys=False
)
# report_long_short_gp = report_long_short_df.groupby(
# [report_long_short_df.index.year, report_long_short_df.index.month]
# [report_long_short_df.index.year, report_long_short_df.index.month], group_keys=False
# )
gp_month = sorted(set(report_normal_gp.size().index))
@@ -97,7 +99,7 @@ def _get_monthly_analysis_with_feature(monthly_df: pd.DataFrame, feature: str =
:param feature:
:return:
"""
_monthly_df_gp = monthly_df.reset_index().groupby(["level_1"])
_monthly_df_gp = monthly_df.reset_index().groupby(["level_1"], group_keys=False)
_name_df = _monthly_df_gp.get_group(feature).set_index(["level_0", "level_1"])
_temp_df = _name_df.pivot_table(index="date", values=["risk"], columns=_name_df.index)

6
qlib/contrib/report/analysis_position/score_ic.py
View File

@@ -15,8 +15,10 @@ def _get_score_ic(pred_label: pd.DataFrame):
"""
concat_data = pred_label.copy()
concat_data.dropna(axis=0, how="any", inplace=True)
_ic = concat_data.groupby(level="datetime").apply(lambda x: x["label"].corr(x["score"]))
_rank_ic = concat_data.groupby(level="datetime").apply(lambda x: x["label"].corr(x["score"], method="spearman"))
_ic = concat_data.groupby(level="datetime", group_keys=False).apply(lambda x: x["label"].corr(x["score"]))
_rank_ic = concat_data.groupby(level="datetime", group_keys=False).apply(
lambda x: x["label"].corr(x["score"], method="spearman")
)
return pd.DataFrame({"ic": _ic, "rank_ic": _rank_ic})

16
qlib/contrib/report/data/ana.py
View File

@@ -72,10 +72,10 @@ class ValueCNT(FeaAnalyser):
self._val_cnt = {}
for col, item in self._dataset.items():
if not super().skip(col):
self._val_cnt[col] = item.groupby(DT_COL_NAME).apply(lambda s: len(s.unique()))
self._val_cnt[col] = item.groupby(DT_COL_NAME, group_keys=False).apply(lambda s: len(s.unique()))
self._val_cnt = pd.DataFrame(self._val_cnt)
if self.ratio:
self._val_cnt = self._val_cnt.div(self._dataset.groupby(DT_COL_NAME).size(), axis=0)
self._val_cnt = self._val_cnt.div(self._dataset.groupby(DT_COL_NAME, group_keys=False).size(), axis=0)
# TODO: transfer this feature to other analysers
ymin, ymax = self._val_cnt.min().min(), self._val_cnt.max().max()
@@ -98,7 +98,7 @@ class FeaInfAna(NumFeaAnalyser):
self._inf_cnt = {}
for col, item in self._dataset.items():
if not super().skip(col):
self._inf_cnt[col] = item.apply(np.isinf).astype(np.int).groupby(DT_COL_NAME).sum()
self._inf_cnt[col] = item.apply(np.isinf).astype(np.int).groupby(DT_COL_NAME, group_keys=False).sum()
self._inf_cnt = pd.DataFrame(self._inf_cnt)
def skip(self, col):
@@ -111,7 +111,7 @@ class FeaInfAna(NumFeaAnalyser):
class FeaNanAna(FeaAnalyser):
def calc_stat_values(self):
self._nan_cnt = self._dataset.isna().groupby(DT_COL_NAME).sum()
self._nan_cnt = self._dataset.isna().groupby(DT_COL_NAME, group_keys=False).sum()
def skip(self, col):
return (col not in self._nan_cnt) or (self._nan_cnt[col].sum() == 0)
@@ -123,8 +123,8 @@ class FeaNanAna(FeaAnalyser):
class FeaNanAnaRatio(FeaAnalyser):
def calc_stat_values(self):
self._nan_cnt = self._dataset.isna().groupby(DT_COL_NAME).sum()
self._total_cnt = self._dataset.groupby(DT_COL_NAME).size()
self._nan_cnt = self._dataset.isna().groupby(DT_COL_NAME, group_keys=False).sum()
self._total_cnt = self._dataset.groupby(DT_COL_NAME, group_keys=False).size()
def skip(self, col):
return (col not in self._nan_cnt) or (self._nan_cnt[col].sum() == 0)
@@ -176,8 +176,8 @@ class FeaSkewTurt(NumFeaAnalyser):
class FeaMeanStd(NumFeaAnalyser):
def calc_stat_values(self):
self._std = self._dataset.groupby(DT_COL_NAME).std()
self._mean = self._dataset.groupby(DT_COL_NAME).mean()
self._std = self._dataset.groupby(DT_COL_NAME, group_keys=False).std()
self._mean = self._dataset.groupby(DT_COL_NAME, group_keys=False).mean()
def plot_single(self, col, ax):
self._mean[col].plot(ax=ax, label="mean")

4
qlib/contrib/strategy/rule_strategy.py
View File

@@ -347,7 +347,7 @@ class SBBStrategyEMA(SBBStrategyBase):
self.signal = {}
if not signal_df.empty:
for stock_id, stock_val in signal_df.groupby(level="instrument"):
for stock_id, stock_val in signal_df.groupby(level="instrument", group_keys=False):
self.signal[stock_id] = stock_val["signal"].droplevel(level="instrument")
def reset_level_infra(self, level_infra):
@@ -434,7 +434,7 @@ class ACStrategy(BaseStrategy):
self.signal = {}
if not signal_df.empty:
for stock_id, stock_val in signal_df.groupby(level="instrument"):
for stock_id, stock_val in signal_df.groupby(level="instrument", group_keys=False):
self.signal[stock_id] = stock_val["volatility"].droplevel(level="instrument")
def reset_level_infra(self, level_infra):