From 79026e5390acd8f719c539e4c467d7f67895dd55 Mon Sep 17 00:00:00 2001
From: chaosyu <chaos.you@gmail.com>
Date: Fri, 25 Jun 2021 09:38:49 +0800
Subject: [PATCH] fix bug that duplicate rows will cause reindex failed when
 dumping with csv files

---
 scripts/dump_bin.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scripts/dump_bin.py b/scripts/dump_bin.py
index 83daa28bc..8e9878895 100644
--- a/scripts/dump_bin.py
+++ b/scripts/dump_bin.py
@@ -244,6 +244,10 @@ class DumpDataBase:
         if df is None or df.empty:
             logger.warning(f"{code} data is None or empty")
             return
+
+        # try to remove dup rows or it will cause exception when reindex.
+        df = df.drop_duplicates(self.date_field_name)
+
         # features save dir
         features_dir = self._features_dir.joinpath(code_to_fname(code).lower())
         features_dir.mkdir(parents=True, exist_ok=True)