liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-16 17:12:20 +08:00
Code Activity

remove redundant file only when remote artifact

Browse Source
This commit is contained in:
Young
2021-11-01 18:55:44 +08:00
parent d929d4bb21
commit 7a884fa9f2
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
qlib/workflow/recorder.py
View File

@@ -11,6 +11,7 @@ from datetime import datetime
from qlib.utils.exceptions import LoadObjectError from qlib.utils.exceptions import LoadObjectError
from ..utils.objm import FileManager from ..utils.objm import FileManager
from ..log import get_module_logger from ..log import get_module_logger
from mlflow.store.artifact.azure_blob_artifact_repo import AzureBlobArtifactRepository
logger = get_module_logger("workflow", logging.INFO) logger = get_module_logger("workflow", logging.INFO)
@@ -335,7 +336,11 @@ class MLflowRecorder(Recorder):
path = self.client.download_artifacts(self.id, name) path = self.client.download_artifacts(self.id, name)
with Path(path).open("rb") as f: with Path(path).open("rb") as f:
data = pickle.load(f) data = pickle.load(f)
os.remove(path) ar = self.client._tracking_client._get_artifact_repo(self.id)
if isinstance(ar, AzureBlobArtifactRepository):
# for saving disk space
# For safety, only remove redundant file for specific ArtifactRepository
shutil.rmtree(Path(path).absolute().parent)
return data return data
except Exception as e: except Exception as e:
raise LoadObjectError(message=str(e)) raise LoadObjectError(message=str(e))