mirror of
https://github.com/microsoft/qlib.git
synced 2026-06-06 05:51:17 +08:00
3097dcc995
* fix(security): enforce RestrictedUnpickler for load_instance to prevent unsafe pickle deserialization * fix: lint error
18 lines
540 B
Python
Executable File
18 lines
540 B
Python
Executable File
import os
|
|
import pandas as pd
|
|
from tqdm import tqdm
|
|
|
|
from qlib.utils.pickle_utils import restricted_pickle_load
|
|
|
|
for tag in ["test", "valid"]:
|
|
files = os.listdir(os.path.join("data/orders/", tag))
|
|
dfs = []
|
|
for f in tqdm(files):
|
|
with open(os.path.join("data/orders/", tag, f), "rb") as fr:
|
|
df = restricted_pickle_load(fr)
|
|
df = df.drop(["$close0"], axis=1)
|
|
dfs.append(df)
|
|
|
|
total_df = pd.concat(dfs)
|
|
pickle.dump(total_df, open(os.path.join("data", "orders", f"{tag}_orders.pkl"), "wb"))
|