liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-16 01:06:56 +08:00
Code Activity

Fix pylint (#888)

Browse Source 
* add_pylint_to_workflow

* fix-pylint

* fix_pylinterror

* fix-issue
This commit is contained in:
SunsetWolf
2022-01-26 19:27:24 +08:00
committed by GitHub
parent 635632e4ed
commit 144e1e2459
103 changed files with 318 additions and 387 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
qlib/contrib/model/gbdt.py
View File

@@ -101,7 +101,7 @@ class LGBModel(ModelFT, LightGBMFInt):
verbose level
"""
# Based on existing model and finetune by train more rounds
dtrain, _ = self._prepare_data(dataset, reweighter)
dtrain, _ = self._prepare_data(dataset, reweighter) # pylint: disable=W0632
if dtrain.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
self.model = lgb.train(

2
qlib/contrib/model/highfreq_gdbt_model.py
View File

@@ -58,7 +58,7 @@ class HFLGBModel(ModelFT, LightGBMFInt):
"""
Test the signal in high frequency test set
"""
if self.model == None:
if self.model is None:
raise ValueError("Model hasn't been trained yet")
df_test = dataset.prepare("test", col_set=["feature", "label"], data_key=DataHandlerLP.DK_I)
df_test.dropna(inplace=True)

36
qlib/contrib/model/pytorch_adarnn.py
View File

@@ -1,12 +1,10 @@
# Copyright (c) Microsoft Corporation.
import os
from pdb import set_trace
from torch.utils.data import Dataset, DataLoader
import copy
from typing import Text, Union
import math
import numpy as np
import pandas as pd
import torch
@@ -182,11 +180,11 @@ class ADARNN(Model):
continue
total_loss = torch.zeros(1).cuda()
for i in range(len(index)):
feature_s = list_feat[index[i][0]]
feature_t = list_feat[index[i][1]]
label_reg_s = list_label[index[i][0]]
label_reg_t = list_label[index[i][1]]
for i, n in enumerate(index):
feature_s = list_feat[n[0]]
feature_t = list_feat[n[1]]
label_reg_s = list_label[n[0]]
label_reg_t = list_label[n[1]]
feature_all = torch.cat((feature_s, feature_t), 0)
if epoch < self.pre_epoch:
@@ -410,7 +408,7 @@ class AdaRNN(nn.Module):
in_size = hidden
self.features = nn.Sequential(*features)
if use_bottleneck == True: # finance
if use_bottleneck is True: # finance
self.bottleneck = nn.Sequential(
nn.Linear(n_hiddens[-1], bottleneck_width),
nn.Linear(bottleneck_width, bottleneck_width),
@@ -449,7 +447,7 @@ class AdaRNN(nn.Module):
def forward_pre_train(self, x, len_win=0):
out = self.gru_features(x)
fea = out[0] # [2N,L,H]
if self.use_bottleneck == True:
if self.use_bottleneck is True:
fea_bottleneck = self.bottleneck(fea[:, -1, :])
fc_out = self.fc(fea_bottleneck).squeeze()
else:
@@ -458,8 +456,8 @@ class AdaRNN(nn.Module):
out_list_all, out_weight_list = out[1], out[2]
out_list_s, out_list_t = self.get_features(out_list_all)
loss_transfer = torch.zeros((1,)).cuda()
for i in range(len(out_list_s)):
criterion_transder = TransferLoss(loss_type=self.trans_loss, input_dim=out_list_s[i].shape[2])
for i, n in enumerate(out_list_s):
criterion_transder = TransferLoss(loss_type=self.trans_loss, input_dim=n.shape[2])
h_start = 0
for j in range(h_start, self.len_seq, 1):
i_start = j - len_win if j - len_win >= 0 else 0
@@ -471,7 +469,7 @@ class AdaRNN(nn.Module):
else 1 / (self.len_seq - h_start) * (2 * len_win + 1)
)
loss_transfer = loss_transfer + weight * criterion_transder.compute(
out_list_s[i][:, j, :], out_list_t[i][:, k, :]
n[:, j, :], out_list_t[i][:, k, :]
)
return fc_out, loss_transfer, out_weight_list
@@ -484,7 +482,7 @@ class AdaRNN(nn.Module):
out, _ = self.features[i](x_input.float())
x_input = out
out_lis.append(out)
if self.model_type == "AdaRNN" and predict == False:
if self.model_type == "AdaRNN" and predict is False:
out_gate = self.process_gate_weight(x_input, i)
out_weight_list.append(out_gate)
return out, out_lis, out_weight_list
@@ -524,10 +522,10 @@ class AdaRNN(nn.Module):
else:
weight = weight_mat
dist_mat = torch.zeros(self.num_layers, self.len_seq).cuda()
for i in range(len(out_list_s)):
criterion_transder = TransferLoss(loss_type=self.trans_loss, input_dim=out_list_s[i].shape[2])
for i, n in enumerate(out_list_s):
criterion_transder = TransferLoss(loss_type=self.trans_loss, input_dim=n.shape[2])
for j in range(self.len_seq):
loss_trans = criterion_transder.compute(out_list_s[i][:, j, :], out_list_t[i][:, j, :])
loss_trans = criterion_transder.compute(n[:, j, :], out_list_t[i][:, j, :])
loss_transfer = loss_transfer + weight[i, j] * loss_trans
dist_mat[i, j] = loss_trans
return fc_out, loss_transfer, dist_mat, weight
@@ -546,7 +544,7 @@ class AdaRNN(nn.Module):
def predict(self, x):
out = self.gru_features(x, predict=True)
fea = out[0]
if self.use_bottleneck == True:
if self.use_bottleneck is True:
fea_bottleneck = self.bottleneck(fea[:, -1, :])
fc_out = self.fc(fea_bottleneck).squeeze()
else:
@@ -572,12 +570,12 @@ class TransferLoss:
Returns:
[tensor] -- transfer loss
"""
if self.loss_type == "mmd_lin" or self.loss_type == "mmd":
if self.loss_type in ("mmd_lin", "mmd"):
mmdloss = MMD_loss(kernel_type="linear")
loss = mmdloss(X, Y)
elif self.loss_type == "coral":
loss = CORAL(X, Y)
elif self.loss_type == "cosine" or self.loss_type == "cos":
elif self.loss_type in ("cosine", "cos"):
loss = 1 - cosine(X, Y)
elif self.loss_type == "kl":
loss = kl_div(X, Y)

1
qlib/contrib/model/pytorch_add.py
View File

@@ -20,7 +20,6 @@ from qlib.contrib.model.pytorch_lstm import LSTMModel
from qlib.contrib.model.pytorch_utils import count_parameters
from qlib.data.dataset import DatasetH
from qlib.data.dataset.handler import DataHandlerLP
from qlib.data.dataset.processor import CSRankNorm
from qlib.log import get_module_logger
from qlib.model.base import Model
from qlib.utils import get_or_create_path

7
qlib/contrib/model/pytorch_alstm.py
View File

@@ -5,7 +5,6 @@
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import pandas as pd
from typing import Text, Union
@@ -150,7 +149,7 @@ class ALSTM(Model):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)
@@ -312,8 +311,8 @@ class ALSTMModel(nn.Module):
def _build_model(self):
try:
klass = getattr(nn, self.rnn_type.upper())
except:
raise ValueError("unknown rnn_type `%s`" % self.rnn_type)
except Exception as e:
raise ValueError("unknown rnn_type `%s`" % self.rnn_type) from e
self.net = nn.Sequential()
self.net.add_module("fc_in", nn.Linear(in_features=self.input_size, out_features=self.hid_size))
self.net.add_module("act", nn.Tanh())

9
qlib/contrib/model/pytorch_alstm_ts.py
View File

@@ -5,7 +5,6 @@
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import pandas as pd
from typing import Text, Union
@@ -20,7 +19,7 @@ from torch.utils.data import DataLoader
from .pytorch_utils import count_parameters
from ...model.base import Model
from ...data.dataset import DatasetH, TSDatasetH
from ...data.dataset import DatasetH
from ...data.dataset.handler import DataHandlerLP
from ...model.utils import ConcatDataset
from ...data.dataset.weight import Reweighter
@@ -160,7 +159,7 @@ class ALSTM(Model):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)
@@ -320,8 +319,8 @@ class ALSTMModel(nn.Module):
def _build_model(self):
try:
klass = getattr(nn, self.rnn_type.upper())
except:
raise ValueError("unknown rnn_type `%s`" % self.rnn_type)
except Exception as e:
raise ValueError("unknown rnn_type `%s`" % self.rnn_type) from e
self.net = nn.Sequential()
self.net.add_module("fc_in", nn.Linear(in_features=self.input_size, out_features=self.hid_size))
self.net.add_module("act", nn.Tanh())

7
qlib/contrib/model/pytorch_gats.py
View File

@@ -5,7 +5,6 @@
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import pandas as pd
from typing import Text, Union
@@ -158,7 +157,7 @@ class GATs(Model):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)
@@ -263,7 +262,9 @@ class GATs(Model):
pretrained_model.load_state_dict(torch.load(self.model_path, map_location=self.device))
model_dict = self.GAT_model.state_dict()
pretrained_dict = {k: v for k, v in pretrained_model.state_dict().items() if k in model_dict}
pretrained_dict = {
k: v for k, v in pretrained_model.state_dict().items() if k in model_dict
} # pylint: disable=E1135
model_dict.update(pretrained_dict)
self.GAT_model.load_state_dict(model_dict)
self.logger.info("Loading pretrained model Done...")

8
qlib/contrib/model/pytorch_gats_ts.py
View File

@@ -5,7 +5,6 @@
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import pandas as pd
import copy
@@ -19,7 +18,6 @@ from torch.utils.data import Sampler
from .pytorch_utils import count_parameters
from ...model.base import Model
from ...data.dataset import DatasetH
from ...data.dataset.handler import DataHandlerLP
from ...contrib.model.pytorch_lstm import LSTMModel
from ...contrib.model.pytorch_gru import GRUModel
@@ -178,7 +176,7 @@ class GATs(Model):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)
@@ -279,7 +277,9 @@ class GATs(Model):
pretrained_model.load_state_dict(torch.load(self.model_path, map_location=self.device))
model_dict = self.GAT_model.state_dict()
pretrained_dict = {k: v for k, v in pretrained_model.state_dict().items() if k in model_dict}
pretrained_dict = {
k: v for k, v in pretrained_model.state_dict().items() if k in model_dict
} # pylint: disable=E1135
model_dict.update(pretrained_dict)
self.GAT_model.load_state_dict(model_dict)
self.logger.info("Loading pretrained model Done...")

3
qlib/contrib/model/pytorch_gru.py
View File

@@ -5,7 +5,6 @@
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import pandas as pd
from typing import Text, Union
@@ -150,7 +149,7 @@ class GRU(Model):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)

4
qlib/contrib/model/pytorch_gru_ts.py
View File

@@ -5,7 +5,6 @@
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import pandas as pd
import copy
@@ -19,7 +18,6 @@ from torch.utils.data import DataLoader
from .pytorch_utils import count_parameters
from ...model.base import Model
from ...data.dataset import DatasetH, TSDatasetH
from ...data.dataset.handler import DataHandlerLP
from ...model.utils import ConcatDataset
from ...data.dataset.weight import Reweighter
@@ -159,7 +157,7 @@ class GRU(Model):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)

7
qlib/contrib/model/pytorch_localformer.py
View File

@@ -5,7 +5,6 @@
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import pandas as pd
from typing import Text, Union
@@ -17,11 +16,9 @@ from ...log import get_module_logger
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from .pytorch_utils import count_parameters
from ...model.base import Model
from ...data.dataset import DatasetH, TSDatasetH
from ...data.dataset import DatasetH
from ...data.dataset.handler import DataHandlerLP
from torch.nn.modules.container import ModuleList
@@ -102,7 +99,7 @@ class LocalformerModel(Model):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)

6
qlib/contrib/model/pytorch_localformer_ts.py
View File

@@ -5,7 +5,6 @@
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import pandas as pd
import copy
@@ -18,9 +17,8 @@ import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from .pytorch_utils import count_parameters
from ...model.base import Model
from ...data.dataset import DatasetH, TSDatasetH
from ...data.dataset import DatasetH
from ...data.dataset.handler import DataHandlerLP
from torch.nn.modules.container import ModuleList
@@ -101,7 +99,7 @@ class LocalformerModel(Model):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)

3
qlib/contrib/model/pytorch_lstm.py
View File

@@ -5,7 +5,6 @@
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import pandas as pd
from typing import Text, Union
@@ -146,7 +145,7 @@ class LSTM(Model):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)

4
qlib/contrib/model/pytorch_lstm_ts.py
View File

@@ -5,7 +5,6 @@
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import pandas as pd
import copy
@@ -18,7 +17,6 @@ import torch.optim as optim
from torch.utils.data import DataLoader
from ...model.base import Model
from ...data.dataset import DatasetH, TSDatasetH
from ...data.dataset.handler import DataHandlerLP
from ...model.utils import ConcatDataset
from ...data.dataset.weight import Reweighter
@@ -155,7 +153,7 @@ class LSTM(Model):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)

3
qlib/contrib/model/pytorch_nn.py
View File

@@ -328,6 +328,7 @@ class Net(nn.Module):
dnn_layers = []
drop_input = nn.Dropout(0.05)
dnn_layers.append(drop_input)
hidden_units = None
for i, (_input_dim, hidden_units) in enumerate(zip(layers[:-1], layers[1:])):
fc = nn.Linear(_input_dim, hidden_units)
activation = nn.LeakyReLU(negative_slope=0.1, inplace=False)
@@ -338,7 +339,7 @@ class Net(nn.Module):
dnn_layers.append(drop_input)
fc = nn.Linear(hidden_units, output_dim)
dnn_layers.append(fc)
# optimizer
# optimizer # pylint: disable=W0631
self.dnn_layers = nn.ModuleList(dnn_layers)
self._weight_init()

3
qlib/contrib/model/pytorch_sfm.py
View File

@@ -4,7 +4,6 @@
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import pandas as pd
from typing import Text, Union
@@ -435,7 +434,7 @@ class SFM(Model):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)

3
qlib/contrib/model/pytorch_tabnet.py
View File

@@ -3,7 +3,6 @@
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import pandas as pd
from typing import Text, Union
@@ -378,7 +377,7 @@ class TabnetModel(Model):
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)

3
qlib/contrib/model/pytorch_tcn.py
View File

@@ -15,7 +15,6 @@ from ...log import get_module_logger
import torch
import torch.nn as nn
import torch.optim as optim
from torch.nn.utils import weight_norm
from .pytorch_utils import count_parameters
from ...model.base import Model
@@ -158,7 +157,7 @@ class TCN(Model):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)

2
qlib/contrib/model/pytorch_tcn_ts.py
View File

@@ -158,7 +158,7 @@ class TCN(Model):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)

14
qlib/contrib/model/pytorch_tcts.py
View File

@@ -5,20 +5,12 @@
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import pandas as pd
import copy
import random
from sklearn.metrics import roc_auc_score, mean_squared_error
import logging
from ...utils import (
unpack_archive_with_buffer,
save_multiple_parts_file,
get_or_create_path,
drop_nan_by_y_index,
)
from ...log import get_module_logger, TimeInspector
from ...utils import get_or_create_path
from ...log import get_module_logger
import torch
import torch.nn as nn
@@ -263,7 +255,7 @@ class TCTS(Model):
x_valid, y_valid = df_valid["feature"], df_valid["label"]
x_test, y_test = df_test["feature"], df_test["label"]
if save_path == None:
if save_path is None:
save_path = get_or_create_path(save_path)
best_loss = np.inf
while best_loss > self.lowest_valid_performance:

3
qlib/contrib/model/pytorch_tra.py
View File

@@ -6,10 +6,8 @@ import os
import copy
import math
import json
import collections
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import torch
@@ -24,7 +22,6 @@ except ImportError:
from tqdm import tqdm
from qlib.utils import get_or_create_path
from qlib.constant import EPS
from qlib.log import get_module_logger
from qlib.model.base import Model

7
qlib/contrib/model/pytorch_transformer.py
View File

@@ -5,7 +5,6 @@
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import pandas as pd
from typing import Text, Union
@@ -17,11 +16,9 @@ from ...log import get_module_logger
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from .pytorch_utils import count_parameters
from ...model.base import Model
from ...data.dataset import DatasetH, TSDatasetH
from ...data.dataset import DatasetH
from ...data.dataset.handler import DataHandlerLP
# qrun examples/benchmarks/Transformer/workflow_config_transformer_Alpha360.yaml ”
@@ -101,7 +98,7 @@ class TransformerModel(Model):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)

6
qlib/contrib/model/pytorch_transformer_ts.py
View File

@@ -5,7 +5,6 @@
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import pandas as pd
import copy
@@ -18,9 +17,8 @@ import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from .pytorch_utils import count_parameters
from ...model.base import Model
from ...data.dataset import DatasetH, TSDatasetH
from ...data.dataset import DatasetH
from ...data.dataset.handler import DataHandlerLP
@@ -98,7 +96,7 @@ class TransformerModel(Model):
mask = torch.isfinite(label)
if self.metric == "" or self.metric == "loss":
if self.metric in ("", "loss"):
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)

6
qlib/contrib/model/pytorch_utils.py
View File

@@ -26,11 +26,11 @@ def count_parameters(models_or_parameters, unit="m"):
else:
counts = sum(v.numel() for v in models_or_parameters)
unit = unit.lower()
if unit == "kb" or unit == "k":
if unit in ("kb", "k"):
counts /= 2 ** 10
elif unit == "mb" or unit == "m":
elif unit in ("mb", "m"):
counts /= 2 ** 20
elif unit == "gb" or unit == "g":
elif unit in ("gb", "g"):
counts /= 2 ** 30
elif unit is not None:
raise ValueError("Unknown unit: {:}".format(unit))

1
qlib/contrib/model/tcn.py
View File

@@ -1,6 +1,5 @@
# MIT License
# Copyright (c) 2018 CMU Locus Lab
import torch
import torch.nn as nn
from torch.nn.utils import weight_norm