diff --git a/mmdet/datasets/custom.py b/mmdet/datasets/custom.py
index 85feba8f2ef..c895bcfecff 100644
--- a/mmdet/datasets/custom.py
+++ b/mmdet/datasets/custom.py
@@ -81,7 +81,7 @@ def __init__(self,
         # load annotations (and proposals)
         self.data_infos = self.load_annotations(self.ann_file)
         # filter data infos if classes are customized
-        if self.custom_classes:
+        if self.custom_classes and not self.test_mode:
             self.data_infos = self.get_subset_by_classes()
 
         if self.proposal_file is not None: