forked from treeverse/dvc
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy path__init__.py
More file actions
45 lines (34 loc) 路 1.12 KB
/
Copy path__init__.py
File metadata and controls
45 lines (34 loc) 路 1.12 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
import logging
from concurrent.futures import ThreadPoolExecutor, as_completed
from dvc.progress import Tqdm
from .tree import Tree
logger = logging.getLogger(__name__)
def save(odb, obj, jobs=None, **kwargs):
if isinstance(obj, Tree):
with ThreadPoolExecutor(max_workers=jobs) as executor:
for future in Tqdm(
as_completed(
executor.submit(
odb.add,
entry.path_info,
entry.fs,
entry.hash_info,
**kwargs
)
for _, entry in obj
),
desc="Saving files",
total=len(obj),
unit="file",
):
future.result()
odb.add(obj.path_info, obj.fs, obj.hash_info, **kwargs)
def check(odb, obj):
odb.check(obj.hash_info)
if isinstance(obj, Tree):
for _, entry in obj:
odb.check(entry.hash_info)
def load(odb, hash_info):
if hash_info.isdir:
return Tree.load(odb, hash_info)
return odb.get(hash_info)